From 0dc862211c7e6f6d281a306d1a8e968cbbfd5554 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 27 Jan 2026 15:42:14 -0500 Subject: [PATCH 001/119] #697 Add fre workflow checkout tool and test --- fre/fre.py | 3 +- fre/workflow/README.md | 0 fre/workflow/__init__.py | 27 ++++ fre/workflow/checkout_script.py | 120 ++++++++++++++ fre/workflow/freworkflow.py | 101 ++++++++++++ fre/workflow/tests/AM5_example/am5.yaml | 71 +++++++++ .../yaml_include/pp-test.c96_amip.yaml | 33 ++++ .../AM5_example/yaml_include/pp.c96_amip.yaml | 94 +++++++++++ .../AM5_example/yaml_include/settings.yaml | 32 ++++ .../yaml_include/settings_WRONG.yaml | 32 ++++ fre/workflow/tests/test_checkout_script.py | 147 ++++++++++++++++++ 11 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 fre/workflow/README.md create mode 100644 fre/workflow/__init__.py create mode 100644 fre/workflow/checkout_script.py create mode 100644 fre/workflow/freworkflow.py create mode 100644 fre/workflow/tests/AM5_example/am5.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/settings.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml create mode 100644 fre/workflow/tests/test_checkout_script.py diff --git a/fre/fre.py b/fre/fre.py index 25b717504..2c60308d0 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -22,7 +22,8 @@ # click and lazy group loading @click.group( cls = LazyGroup, - lazy_subcommands = {"pp": ".pp.frepp.pp_cli", + lazy_subcommands = {"workflow": ".workflow.freworkflow.workflow_cli", + "pp": ".pp.frepp.pp_cli", "catalog": ".catalog.frecatalog.catalog_cli", "list": ".list_.frelist.list_cli", "check": ".check.frecheck.check_cli", diff --git a/fre/workflow/README.md b/fre/workflow/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/fre/workflow/__init__.py b/fre/workflow/__init__.py new file mode 100644 index 000000000..dc62bb794 --- /dev/null +++ b/fre/workflow/__init__.py @@ -0,0 +1,27 @@ +from typing import Optional + +def make_workflow_name(experiment : Optional[str] = None) -> str: + """ + Function that takes in a triplet of tags for a model experiment, platform, and target, and + returns a directory name for the corresponding pp workflow. Because this is often given by + user to the shell being used by python, we split/reform the string to remove semi-colons or + spaces that may be used to execute an arbitrary command with elevated privileges. + + :param experiment: One of the postprocessing experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None + :type experiment: str + :param platform: The location + compiler that was used to run the model (e.g. gfdl.ncrc5-deploy), default None + :type platform: str + :param target: Options used for the model compiler (e.g. prod-openmp), default None + :type target: str + :return: string created in specific format from the input strings + :rtype: str + + .. note:: if any arguments are None, then "None" will appear in the workflow name + """ + name = f'{experiment}__{platform}__{target}' + return ''.join( + (''.join( + name.split(' ') + ) + ).split(';') + ) # user-input sanitation, prevents some malicious cmds from being executed with privileges diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py new file mode 100644 index 000000000..58539b1fd --- /dev/null +++ b/fre/workflow/checkout_script.py @@ -0,0 +1,120 @@ +""" Workflow checkout """ +import os +import subprocess +from pathlib import Path +import logging + +import fre.yamltools.combine_yamls_script as cy +from fre.app.helpers import change_directory +#from . import make_workflow_name + +fre_logger = logging.getLogger(__name__) + +FRE_PP_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" +FRE_RUN_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" + +def workflow_checkout(yamlfile: str = None, experiment = None, + application = None, branch = None): + """ + Create a directory and clone the workflow template files from a defined repo. + + :param yamlfile: Model yaml configuration file + :type yamlfile: str + :param experiment: One of the postprocessing experiment names from the + yaml displayed by fre list exps -y $yamlfile + (e.g. c96L65_am5f4b4r0_amip), default None + :type experiment: str + :param platform: The location + compiler that was used to run the model + (e.g. gfdl.ncrc5-deploy), default None + :type platform: str + :param target: Options used for the model compiler (e.g. prod-openmp), default None + :type target: str + :param branch: which git branch to pull from, default None + :type branch: str + :param application: Which workflow will be used/cloned + :type application: str + :raises OSError: why checkout script was not able to be created + :raises ValueError: + -if experiment or platform or target is None + -if branch argument cannot be found as a branch or tag + """ + # Used in consolidate_yamls function for now + platform = None + target = None + if application == "run": + fre_logger.info("NOT DONE YET") + # will probably be taken out and put above is "use" + # is generalized in this tool + yaml = cy.consolidate_yamls(yamlfile=yamlfile, + experiment=experiment, + platform=platform, + target=target, + use="run", + output=None) + workflow_info = yaml.get("workflow").get("run_workflow") + elif application == "pp": + # will probably be taken out and put above is "use" + # is generalized in this tool + yaml = cy.consolidate_yamls(yamlfile=yamlfile, + experiment=experiment, + platform=platform, + target=target, + use="pp", + output=None) + workflow_info = yaml.get("workflow").get("pp_workflow") + + repo = workflow_info.get("repo") + + if not branch: + tag = workflow_info.get("version") + fre_logger.info("Default tag ==> '%s'", tag) + else: + tag = branch + fre_logger.info("Requested branch/tag ==> '%s'", tag) + + if None in [repo, tag]: + raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}") + + fre_logger.warning("(%s):(%s) check out ==> REQUESTED", repo, tag) + + # clone directory + directory = os.path.expanduser("~/cylc-src") + # workflow name + workflow_name = experiment + + # create workflow in cylc-src + try: + Path(directory).mkdir(parents=True, exist_ok=True) + except Exception as exc: + raise OSError( + f"(checkoutScript) directory {directory} wasn't able to be created. exit!") from exc + + if not Path(f"{directory}/{workflow_name}").is_dir(): + # scenarios 1+2, checkout doesn't exist, branch specified (or not) + fre_logger.info("Workflow does not yet exist; will create now") + clone_output = subprocess.run( ["git", "clone","--recursive", + f"--branch={tag}", + repo, f"{directory}/{workflow_name}"], + capture_output = True, text = True, check = True) + fre_logger.debug(clone_output) + fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + else: + # the repo checkout does exist, scenarios 3 and 4. + with change_directory(f"{directory}/{workflow_name}"): + # capture the branch and tag + # if either match git_clone_branch_arg, then success. otherwise, fail. + current_tag = subprocess.run(["git","describe","--tags"], + capture_output = True, + text = True, check = True).stdout.strip() + current_branch = subprocess.run(["git", "branch", "--show-current"], + capture_output = True, + text = True, check = True).stdout.strip() + + if tag in (current_tag, current_branch): + fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", directory, workflow_name, tag) + else: + fre_logger.error( + "ERROR: Checkout exists ('%s/%s') and does not match '%s'", directory, workflow_name, tag) + fre_logger.error( + "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) + raise ValueError('Neither tag nor branch matches the git clone branch arg') diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py new file mode 100644 index 000000000..21b6fec44 --- /dev/null +++ b/fre/workflow/freworkflow.py @@ -0,0 +1,101 @@ +''' fre workflow ''' + +import click +import logging +fre_logger = logging.getLogger(__name__) + +#fre tools +from . import checkout_script +from . import install_script +from . import run_script + +@click.group(help=click.style(" - workflow subcommands", fg=(57,139,210))) +def workflow_cli(): + ''' entry point to fre workflow click commands ''' + +@workflow_cli.command() +@click.option("-y", "--yamlfile", type=str, + help="Model yaml file", + required=True) +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +#@click.option("-p", "--platform", type=str, +# help="Platform name") +#@click.option("-t", "--target", type=str, +# help="Target name") +@click.option("-b", "--branch", type =str, + required=False, default = None, + help="fre-workflows branch/tag to clone; default is $(fre --version)") +@click.option("-a", "--application", + type=click.Choice(['run', 'pp']), + help="Use case for checked out workflow", + required=True) +def checkout(yamlfile, experiment, application, branch=None): + """ + Checkout/extract fre workflow + """ + checkout_script.workflow_checkout(yamlfile, experiment, application, branch) + +@workflow_cli.command() +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +#@click.option("-p", "--platform", type=str, +# help="Platform name", +# required=True) +#@click.option("-t", "--target", type=str, +# help="Target name", +# required=True) +def install(experiment): + """ + Install workflow configuration + """ + install_script.workflow_install(experiment) + +@workflow_cli.command() +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +#@click.option("-p", "--platform", type=str, +# help="Platform name", +# required=True) +#@click.option("-t", "--target", type=str, +# help="Target name", +# required=True) +@click.option("--pause", is_flag=True, default=False, + help="Pause the workflow immediately on start up", + required=False) +@click.option("--no_wait", is_flag=True, default=False, + help="after submission, do not wait to ping the scheduler and confirm success", + required=False) +def run(experiment, pause, no_wait): + """ + Run workflow configuration + """ + run_script.workflow_run(experiment, pause, no_wait) + +@workflow_cli.command() +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +#@click.option("-p", "--platform", type=str, +# help="Platform name", +# required=True) +#@click.option("-T", "--target", type=str, +# help="Target name", +# required=True) +@click.option("-c", "--config-file", type=str, + help="Path to a configuration file in either XML or YAML", + required=True) +@click.option("-b", "--branch", + required=False, default=None, + help="fre-workflows branch/tag to clone; default is $(fre --version)") +@click.option("-t", "--time", + required=False, default=None, + help="Time whose history files are ready") +def all(experiment, platform, target, config_file, branch, time): + """ + Execute all fre workflow initialization steps in order + """ + wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) diff --git a/fre/workflow/tests/AM5_example/am5.yaml b/fre/workflow/tests/AM5_example/am5.yaml new file mode 100644 index 000000000..f40d51a0a --- /dev/null +++ b/fre/workflow/tests/AM5_example/am5.yaml @@ -0,0 +1,71 @@ +# reusable variables +fre_properties: + - &AM5_VERSION "am5f7b12r1" + - &FRE_STEM !join [am5/, *AM5_VERSION] + + # amip + - &EXP_AMIP_START "19790101T0000Z" + - &EXP_AMIP_END "20200101T0000Z" + - &ANA_AMIP_START "19800101T0000Z" + - &ANA_AMIP_END "20200101T0000Z" + + - &PP_AMIP_CHUNK96 "P1Y" + - &PP_AMIP_CHUNK384 "P1Y" + - &PP_XYINTERP96 "180,288" + - &PP_XYINTERP384 "720,1152" + + # climo + - &EXP_CLIMO_START96 "0001" + - &EXP_CLIMO_END96 "0011" + - &ANA_CLIMO_START96 "0002" + - &ANA_CLIMO_END96 "0011" + + - &EXP_CLIMO_START384 "0001" + - &EXP_CLIMO_END384 "0006" + - &ANA_CLIMO_START384 "0002" + - &ANA_CLIMO_END384 "0006" + + # coupled + - &PP_CPLD_CHUNK_A "P5Y" + - &PP_CPLD_CHUNK_B "P20Y" + + # grids + - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + - &release "f1a1r1" + - &INTEL "intel-classic" + - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +# compile information +build: + compileYaml: "compile.yaml" + platformYaml: "yaml_include/platforms.yaml" + +experiments: + - name: "c96L65_am5f7b12r1_amip_TESTING" + settings: "yaml_include/settings.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + - "yaml_include/pp-test.c96_amip.yaml" + - name: "c96L65_am5f7b12r1_amip_TESTING_WRONG" + settings: "yaml_include/settings_WRONG.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + - name: "c96L65_am5f7b12r1_pdclim1850F" + pp: + - "yaml_include/pp.c96_clim.yaml" + +# amip: +# settings: +# - shared/settings.yaml +# - shared/directories.yaml +# run: +# version: 1.1 +# - run/inputs.yaml +# - run/runtime.yaml +# postprocess: +# version: 2.0 +# - pp/components +# - analysis/legacy-bw.yaml diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml new file mode 100644 index 000000000..76f078523 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml @@ -0,0 +1,33 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "200,200" + +#c96_amip_postprocess: +postprocess: + components: + - type: "atmos_cmip-TEST" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos-TEST" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level_cmip-TEST" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..d20d05757 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,94 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "180,360" + +#c96_amip_postprocess: +postprocess: + # main pp instructions + components: + - type: "atmos_cmip" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + - type: "atmos_level_cmip" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_month_aer" + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_diurnal" + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_scalar" + sources: + - history_file: "atmos_scalar" + postprocess_on: True + - type: "aerosol_cmip" + xyInterp: *PP_XYINTERP96 + sources: + - history_file: "aerosol_month_cmip" + sourceGrid: "cubedsphere" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "land" + sources: + - history_file: "land_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "land_cmip" + sources: + - history_file: "land_month_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "tracer_level" + sources: + - history_file: "atmos_tracer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml new file mode 100644 index 000000000..791c0c437 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -0,0 +1,32 @@ +#workflow repositories +workflow: + run_workflow: + repo: "https://github.com/NOAA-GFDL" + version: "tbd" + pp_workflow: + repo: "https://github.com/NOAA-GFDL/fre-workflows.git" + version: "main" + +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml new file mode 100644 index 000000000..c973591e8 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -0,0 +1,32 @@ +#workflow repositories +workflow: + run_workflow: + repo: "https://github.com/NOAA-GFDL" + version: "tbd" + pp_workflow: + repo: + version: "main" + +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py new file mode 100644 index 000000000..997e7c5ab --- /dev/null +++ b/fre/workflow/tests/test_checkout_script.py @@ -0,0 +1,147 @@ +''' fre workflow checkout tests ''' +import stat +import re +import os +from pathlib import Path +import pytest +from fre.workflow import checkout_script + +TEST_CONFIGS = "fre/workflow/tests/AM5_example/" +EXPERIMENT = "c96L65_am5f7b12r1_amip_TESTING" + +@pytest.fixture(autouse=True, name="fake_home") +def fake_home_fixture(tmp_path, monkeypatch): + """ + Set the tmp_path as HOME for the cylc-src directory + to be created in. + """ + ## Mock HOME for cylc-src and cylc-run + fake_home = Path(tmp_path) + monkeypatch.setenv("HOME", str(fake_home)) + + return fake_home + +def test_cylc_src_creation_fail(fake_home): + """ + Test for the expected failure if the cylc-src + directory cannot be created. + + This test simulates a permission error in HOME. + """ + # Temporarily change fake_home permissions to read-only, so cylc-src creation will fail + os.chmod(fake_home, stat.S_IREAD) + + # run checkout to create cylc-src + directory = f"{fake_home}/cylc-src" + expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" + with pytest.raises(OSError, match = re.escape(expected_error)): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + +def test_check_missing_repo(): + """ + Test for the expected ValueError if the repo is not + defined in the settings.yaml. + """ + experiment = "c96L65_am5f7b12r1_amip_TESTING_WRONG" + repo = None + tag = "main" + with pytest.raises(ValueError, match = f"One of these are None: repo / tag = {repo} / {tag}"): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = experiment, + application = "pp", + branch = None) + +#def test_run_workflow_checkout(caplog): +# """ +# Test for a successful run workflow checkout. +# """ +# checkout_script.workflow_checkout(yamlfile, +# experiment = "c96L65_am5f7b12r1_amip_TESTING", +# application = "run" +# branch = None) + +def test_pp_workflow_checkout(fake_home, caplog): + """ + Test for a successful post-processing workflow checkout. + """ + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + + expected_repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + expected_tag = "main" + assert all([Path(f"{fake_home}/cylc-src/{EXPERIMENT}").exists(), + Path(f"{fake_home}/cylc-src/{EXPERIMENT}").is_dir(), + Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), + Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").is_file(), + f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) + +def test_pp_workflow_checkout_exists_already(fake_home, caplog): + """ + Test for the expected output message if the checkout/branch already exists. + """ + # 1st checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + + # 2nd checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + + repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + expected_tag = "main" + expected_output = [ + f"({repo}):({expected_tag}) check out ==> REQUESTED", + (f"Checkout exists ('{fake_home}/cylc-src/c96L65_am5f7b12r1_amip_TESTING'), " + f"and matches '{expected_tag}'") + ] + + for string in expected_output: + assert string in caplog.text + +def test_pp_workflow_checkout_branch_override(caplog): + """ + Test for correct checkout if a '-b', '--branch' is specified. + """ + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = "2025.04") + + repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + expected_tag = "2025.04" + expected_output = [f"({repo}):({expected_tag}) check out ==> REQUESTED", + f"({repo}):({expected_tag}) check out ==> SUCCESSFUL"] + + for string in expected_output: + assert string in caplog.text + +def test_pp_workflow_checkout_branch_conflict(fake_home, caplog): + """ + Test for the expected ValueError if the checkout was done already, + but user is checking out same repo again, with a different branch/tag. + """ + # 1st checkout: using default 'main' branch as set in the settings.yaml (version) + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + + expected_output = (f"ERROR: Checkout exists ('{fake_home}/cylc-src/{EXPERIMENT}') " + "and does not match '2025.04'") + expected_error = "Neither tag nor branch matches the git clone branch arg" + with pytest.raises(ValueError, match = expected_error): + # 2nd checkout: trying to check out 2025.04 tag while 'main' already checked out + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = "2025.04") + assert expected_output in caplog.text From 099342ffb766229b73dd5f33a8d1a7398a3b22d2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 27 Jan 2026 15:44:32 -0500 Subject: [PATCH 002/119] #697 Comment out non-existent tools for now --- fre/workflow/freworkflow.py | 98 ++++++++++++++----------------------- 1 file changed, 38 insertions(+), 60 deletions(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 21b6fec44..3fb4d57ba 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -20,10 +20,6 @@ def workflow_cli(): @click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -#@click.option("-p", "--platform", type=str, -# help="Platform name") -#@click.option("-t", "--target", type=str, -# help="Target name") @click.option("-b", "--branch", type =str, required=False, default = None, help="fre-workflows branch/tag to clone; default is $(fre --version)") @@ -37,65 +33,47 @@ def checkout(yamlfile, experiment, application, branch=None): """ checkout_script.workflow_checkout(yamlfile, experiment, application, branch) -@workflow_cli.command() -@click.option("-e", "--experiment", type=str, - help="Experiment name", - required=True) -#@click.option("-p", "--platform", type=str, -# help="Platform name", -# required=True) -#@click.option("-t", "--target", type=str, -# help="Target name", +#@workflow_cli.command() +#@click.option("-e", "--experiment", type=str, +# help="Experiment name", # required=True) -def install(experiment): - """ - Install workflow configuration - """ - install_script.workflow_install(experiment) +#def install(experiment): +# """ +# Install workflow configuration +# """ +# install_script.workflow_install(experiment) -@workflow_cli.command() -@click.option("-e", "--experiment", type=str, - help="Experiment name", - required=True) -#@click.option("-p", "--platform", type=str, -# help="Platform name", +#@workflow_cli.command() +#@click.option("-e", "--experiment", type=str, +# help="Experiment name", # required=True) -#@click.option("-t", "--target", type=str, -# help="Target name", -# required=True) -@click.option("--pause", is_flag=True, default=False, - help="Pause the workflow immediately on start up", - required=False) -@click.option("--no_wait", is_flag=True, default=False, - help="after submission, do not wait to ping the scheduler and confirm success", - required=False) -def run(experiment, pause, no_wait): - """ - Run workflow configuration - """ - run_script.workflow_run(experiment, pause, no_wait) +#@click.option("--pause", is_flag=True, default=False, +# help="Pause the workflow immediately on start up", +# required=False) +#@click.option("--no_wait", is_flag=True, default=False, +# help="after submission, do not wait to ping the scheduler and confirm success", +# required=False) +#def run(experiment, pause, no_wait): +# """ +# Run workflow configuration +# """ +# run_script.workflow_run(experiment, pause, no_wait) -@workflow_cli.command() -@click.option("-e", "--experiment", type=str, - help="Experiment name", - required=True) -#@click.option("-p", "--platform", type=str, -# help="Platform name", +#@workflow_cli.command() +#@click.option("-e", "--experiment", type=str, +# help="Experiment name", # required=True) -#@click.option("-T", "--target", type=str, -# help="Target name", +#@click.option("-c", "--config-file", type=str, +# help="Path to a configuration file in either XML or YAML", # required=True) -@click.option("-c", "--config-file", type=str, - help="Path to a configuration file in either XML or YAML", - required=True) -@click.option("-b", "--branch", - required=False, default=None, - help="fre-workflows branch/tag to clone; default is $(fre --version)") -@click.option("-t", "--time", - required=False, default=None, - help="Time whose history files are ready") -def all(experiment, platform, target, config_file, branch, time): - """ - Execute all fre workflow initialization steps in order - """ - wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) +#@click.option("-b", "--branch", +# required=False, default=None, +# help="fre-workflows branch/tag to clone; default is $(fre --version)") +#@click.option("-t", "--time", +# required=False, default=None, +# help="Time whose history files are ready") +#def all(experiment, platform, target, config_file, branch, time): +# """ +# Execute all fre workflow initialization steps in order +# """ +# wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) From 40d43a5b61211ae4e24fed0bea9e93eeb15f74c1 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 6 Feb 2026 14:21:50 -0500 Subject: [PATCH 003/119] #697 Comment out things that aren't done yet --- fre/workflow/freworkflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 3fb4d57ba..87bdee045 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -6,8 +6,8 @@ #fre tools from . import checkout_script -from . import install_script -from . import run_script +#from . import install_script +#from . import run_script @click.group(help=click.style(" - workflow subcommands", fg=(57,139,210))) def workflow_cli(): From 440a040f4c98cb717a619610331192cac12476e4 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 6 Feb 2026 14:22:16 -0500 Subject: [PATCH 004/119] #697 Add validation and update output file name for resolved yaml --- fre/workflow/checkout_script.py | 53 ++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 58539b1fd..cec0ebf85 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -3,16 +3,59 @@ import subprocess from pathlib import Path import logging +import shutil import fre.yamltools.combine_yamls_script as cy from fre.app.helpers import change_directory #from . import make_workflow_name +from jsonschema import validate, SchemaError, ValidationError fre_logger = logging.getLogger(__name__) FRE_PP_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" FRE_RUN_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" +######VALIDATE##### +def validate_yaml(yamlfile: dict, application: str) -> None: + """ + Validate the format of the yaml file based + on the schema.json in gfdl_msd_schemas + + :param yamlfile: Model, settings, pp, and analysis yaml + information combined into a dictionary + :type yamlfile: dict + :param application: ------------------------------------------------ + :type application: string + :raises ValueError: + - if gfdl_mdf_schema path is not valid + - combined yaml is not valid + - unclear error in validation + :return: None + :rtype: None + """ + schema_dir = Path(__file__).resolve().parents[1] + schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json') + fre_logger.info("Using yaml schema '%s'", schema_path) + # Load the json schema: .load() (vs .loads()) reads and parses the json in one) + try: + with open(schema_path,'r', encoding='utf-8') as s: + schema = json.load(s) + except: + fre_logger.error("Schema '%s' is not valid. Contact the FRE team.", schema_path) + raise + + # Validate yaml + # If the yaml is not valid, the schema validation will raise errors and exit + try: + validate(instance = yamlfile,schema=schema) + fre_logger.info("Combined yaml valid") + except SchemaError as exc: + raise ValueError(f"Schema '{schema_path}' is not valid. Contact the FRE team.") from exc + except ValidationError as exc: + raise ValueError("Combined yaml is not valid. Please fix the errors and try again.") from exc + except Exception as exc: + raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc + def workflow_checkout(yamlfile: str = None, experiment = None, application = None, branch = None): """ @@ -51,6 +94,7 @@ def workflow_checkout(yamlfile: str = None, experiment = None, target=target, use="run", output=None) + #validate_yaml(yamlfile = yaml, application = "run") workflow_info = yaml.get("workflow").get("run_workflow") elif application == "pp": # will probably be taken out and put above is "use" @@ -60,7 +104,8 @@ def workflow_checkout(yamlfile: str = None, experiment = None, platform=platform, target=target, use="pp", - output=None) + output=f"config.yaml") + #validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp_workflow") repo = workflow_info.get("repo") @@ -118,3 +163,9 @@ def workflow_checkout(yamlfile: str = None, experiment = None, fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') + + ## Move combined yaml to cylc-src location + cylc_src_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{experiment}") +# outfile = os.path.join(cylc_src_dir, f"{experiment}.yaml") + shutil.move(f"config.yaml", cylc_src_dir) + fre_logger.info("Combined yaml file moved to ~/cylc-src/%s", experiment) From 3249384ed9f06b8617947bfeb5c79eb745b1a58a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 6 Feb 2026 14:41:58 -0500 Subject: [PATCH 005/119] #697 Add assert to test --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 997e7c5ab..7ef182aaa 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -77,7 +77,7 @@ def test_pp_workflow_checkout(fake_home, caplog): assert all([Path(f"{fake_home}/cylc-src/{EXPERIMENT}").exists(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}").is_dir(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").is_file(), + Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists() f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) def test_pp_workflow_checkout_exists_already(fake_home, caplog): From 7c96c60e5d010e09b57c1673d666298714c8cf7b Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Wed, 11 Feb 2026 10:43:32 -0500 Subject: [PATCH 006/119] Add comma --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 7ef182aaa..138492b45 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -77,7 +77,7 @@ def test_pp_workflow_checkout(fake_home, caplog): assert all([Path(f"{fake_home}/cylc-src/{EXPERIMENT}").exists(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}").is_dir(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists() + Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists(), f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) def test_pp_workflow_checkout_exists_already(fake_home, caplog): From b217d5ed79127d2adc054b7f77788fd7ca8b4bf1 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 14:23:01 -0500 Subject: [PATCH 007/119] #697 Add comma --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 7ef182aaa..138492b45 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -77,7 +77,7 @@ def test_pp_workflow_checkout(fake_home, caplog): assert all([Path(f"{fake_home}/cylc-src/{EXPERIMENT}").exists(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}").is_dir(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists() + Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists(), f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) def test_pp_workflow_checkout_exists_already(fake_home, caplog): From 13ca73ccc0eba3bc3a3e118e3ea310b5bb689cd1 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 14:52:59 -0500 Subject: [PATCH 008/119] #697 Put file moving in checkout creation --- fre/workflow/checkout_script.py | 12 ++++++------ fre/workflow/tests/test_checkout_script.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index cec0ebf85..68bd45c4d 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -143,6 +143,12 @@ def workflow_checkout(yamlfile: str = None, experiment = None, capture_output = True, text = True, check = True) fre_logger.debug(clone_output) fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + + ## Move combined yaml to cylc-src location + cylc_src_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{experiment}") + current_dir = Path.cwd() + shutil.move(Path(f"{current_dir}/config.yaml"), cylc_src_dir) + fre_logger.info("Combined yaml file moved to ~/cylc-src/%s", experiment) else: # the repo checkout does exist, scenarios 3 and 4. with change_directory(f"{directory}/{workflow_name}"): @@ -163,9 +169,3 @@ def workflow_checkout(yamlfile: str = None, experiment = None, fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') - - ## Move combined yaml to cylc-src location - cylc_src_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{experiment}") -# outfile = os.path.join(cylc_src_dir, f"{experiment}.yaml") - shutil.move(f"config.yaml", cylc_src_dir) - fre_logger.info("Combined yaml file moved to ~/cylc-src/%s", experiment) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 138492b45..3039ec21b 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -32,7 +32,7 @@ def test_cylc_src_creation_fail(fake_home): os.chmod(fake_home, stat.S_IREAD) # run checkout to create cylc-src - directory = f"{fake_home}/cylc-src" + directory = Path(f"{fake_home}/cylc-src") expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" with pytest.raises(OSError, match = re.escape(expected_error)): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", From 70171a6249caf62147115495f646c475e7f85eef Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 15:20:54 -0500 Subject: [PATCH 009/119] #697 See if pipeline run as root --- fre/workflow/tests/test_checkout_script.py | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 3039ec21b..884f0c6cc 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -31,14 +31,21 @@ def test_cylc_src_creation_fail(fake_home): # Temporarily change fake_home permissions to read-only, so cylc-src creation will fail os.chmod(fake_home, stat.S_IREAD) - # run checkout to create cylc-src - directory = Path(f"{fake_home}/cylc-src") - expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" - with pytest.raises(OSError, match = re.escape(expected_error)): - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - branch = None) + try: + if os.geteuid == 0: # if running as root + pytest.skip("Cannot test premission errors when running as root") + print("RUNNING AS ROOT") + +# # run checkout to create cylc-src +# directory = Path(f"{fake_home}/cylc-src") +# expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" +# with pytest.raises(OSError, match = re.escape(expected_error)): +# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", +# experiment = EXPERIMENT, +# application = "pp", +# branch = None) + finally: + os.chmod(fake_home, stat.IRWXU) def test_check_missing_repo(): """ From c558027cbb4980f32ca32c2092ebc415300cf63d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 16:23:18 -0500 Subject: [PATCH 010/119] #697 Fix stat --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 884f0c6cc..47a34bd0c 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -45,7 +45,7 @@ def test_cylc_src_creation_fail(fake_home): # application = "pp", # branch = None) finally: - os.chmod(fake_home, stat.IRWXU) + os.chmod(fake_home, stat.S_IRWXU) def test_check_missing_repo(): """ From 60e73ccc7e38329060ce063858555d7f785af4e0 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 17:52:50 -0500 Subject: [PATCH 011/119] #697 make it fail for a sec --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 47a34bd0c..28c3d1018 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -35,7 +35,7 @@ def test_cylc_src_creation_fail(fake_home): if os.geteuid == 0: # if running as root pytest.skip("Cannot test premission errors when running as root") print("RUNNING AS ROOT") - + AH # # run checkout to create cylc-src # directory = Path(f"{fake_home}/cylc-src") # expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" From 38accb6c4eb44f74b1eb9346bc422c0791fcdf8b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:23:48 -0500 Subject: [PATCH 012/119] test failure --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 28c3d1018..c8a654321 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -35,7 +35,7 @@ def test_cylc_src_creation_fail(fake_home): if os.geteuid == 0: # if running as root pytest.skip("Cannot test premission errors when running as root") print("RUNNING AS ROOT") - AH + AH # # run checkout to create cylc-src # directory = Path(f"{fake_home}/cylc-src") # expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" From 06750f2aff0eb4bdbdf571a19c6af882b437da64 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:35:07 -0500 Subject: [PATCH 013/119] #697 Add some readme content --- fre/workflow/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index e69de29bb..51e748026 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -0,0 +1,18 @@ +# FRE workflow +`fre workflow` provides subtools that help to clone, install, and run a workflow from a repository. + +## Quickstart +From the root of the fre-cli repository, run: +``` +# List post-processing experiments defined in model yaml +fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING -a pp +``` + +## Subtools +- `fre workflow checkout [options]` + - Purpose: Clone the workflow repository/branch, depending on the application passed. + - Options: + - `-y, --yamlfile [model yaml] (str; required)` + - `-e, --experiment [experiment name] (str; required)` + - `-b, --branch [workflow repo branch to check out] (str; optional)` + - `-a, --application [ run | pp ] (str; required)` From ecb3ed348d207cf127f1e3f8e2d5487730aca6af Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:36:47 -0500 Subject: [PATCH 014/119] #697 Update comment --- fre/workflow/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 51e748026..121d714c4 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -4,7 +4,7 @@ ## Quickstart From the root of the fre-cli repository, run: ``` -# List post-processing experiments defined in model yaml +# Checkout/clone the post-processing workflow repository fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING -a pp ``` From 32d5906de694e4052a34f569a2f21af8e71188f3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:42:36 -0500 Subject: [PATCH 015/119] #697 Remove click branch overwrite --- fre/workflow/README.md | 1 - fre/workflow/checkout_script.py | 19 +++---------------- fre/workflow/freworkflow.py | 7 ++----- 3 files changed, 5 insertions(+), 22 deletions(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 121d714c4..1006fc61e 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -14,5 +14,4 @@ fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7 - Options: - `-y, --yamlfile [model yaml] (str; required)` - `-e, --experiment [experiment name] (str; required)` - - `-b, --branch [workflow repo branch to check out] (str; optional)` - `-a, --application [ run | pp ] (str; required)` diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 68bd45c4d..222183ae0 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -56,8 +56,7 @@ def validate_yaml(yamlfile: dict, application: str) -> None: except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def workflow_checkout(yamlfile: str = None, experiment = None, - application = None, branch = None): +def workflow_checkout(yamlfile: str = None, experiment = None, application = None): """ Create a directory and clone the workflow template files from a defined repo. @@ -67,19 +66,11 @@ def workflow_checkout(yamlfile: str = None, experiment = None, yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None :type experiment: str - :param platform: The location + compiler that was used to run the model - (e.g. gfdl.ncrc5-deploy), default None - :type platform: str - :param target: Options used for the model compiler (e.g. prod-openmp), default None - :type target: str - :param branch: which git branch to pull from, default None - :type branch: str :param application: Which workflow will be used/cloned :type application: str :raises OSError: why checkout script was not able to be created :raises ValueError: -if experiment or platform or target is None - -if branch argument cannot be found as a branch or tag """ # Used in consolidate_yamls function for now platform = None @@ -110,12 +101,8 @@ def workflow_checkout(yamlfile: str = None, experiment = None, repo = workflow_info.get("repo") - if not branch: - tag = workflow_info.get("version") - fre_logger.info("Default tag ==> '%s'", tag) - else: - tag = branch - fre_logger.info("Requested branch/tag ==> '%s'", tag) + tag = workflow_info.get("version") + fre_logger.info("Defined tag ==> '%s'", tag) if None in [repo, tag]: raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}") diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 87bdee045..ac0964cf4 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -20,18 +20,15 @@ def workflow_cli(): @click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -@click.option("-b", "--branch", type =str, - required=False, default = None, - help="fre-workflows branch/tag to clone; default is $(fre --version)") @click.option("-a", "--application", type=click.Choice(['run', 'pp']), help="Use case for checked out workflow", required=True) -def checkout(yamlfile, experiment, application, branch=None): +def checkout(yamlfile, experiment, application): """ Checkout/extract fre workflow """ - checkout_script.workflow_checkout(yamlfile, experiment, application, branch) + checkout_script.workflow_checkout(yamlfile, experiment, application) #@workflow_cli.command() #@click.option("-e", "--experiment", type=str, From 881366e67a1b080c780231a74e66cdae2eb63a1a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:49:33 -0500 Subject: [PATCH 016/119] #697 Comment out use of branch override --- fre/workflow/tests/test_checkout_script.py | 94 ++++++++++------------ 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index c8a654321..7f4eb711b 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -42,8 +42,7 @@ def test_cylc_src_creation_fail(fake_home): # with pytest.raises(OSError, match = re.escape(expected_error)): # checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", # experiment = EXPERIMENT, -# application = "pp", -# branch = None) +# application = "pp") finally: os.chmod(fake_home, stat.S_IRWXU) @@ -58,8 +57,7 @@ def test_check_missing_repo(): with pytest.raises(ValueError, match = f"One of these are None: repo / tag = {repo} / {tag}"): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = experiment, - application = "pp", - branch = None) + application = "pp") #def test_run_workflow_checkout(caplog): # """ @@ -67,8 +65,7 @@ def test_check_missing_repo(): # """ # checkout_script.workflow_checkout(yamlfile, # experiment = "c96L65_am5f7b12r1_amip_TESTING", -# application = "run" -# branch = None) +# application = "run") def test_pp_workflow_checkout(fake_home, caplog): """ @@ -76,8 +73,7 @@ def test_pp_workflow_checkout(fake_home, caplog): """ checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp", - branch = None) + application = "pp") expected_repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" @@ -94,14 +90,12 @@ def test_pp_workflow_checkout_exists_already(fake_home, caplog): # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp", - branch = None) + application = "pp") # 2nd checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp", - branch = None) + application = "pp") repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" @@ -114,41 +108,41 @@ def test_pp_workflow_checkout_exists_already(fake_home, caplog): for string in expected_output: assert string in caplog.text -def test_pp_workflow_checkout_branch_override(caplog): - """ - Test for correct checkout if a '-b', '--branch' is specified. - """ - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - branch = "2025.04") - - repo = "https://github.com/NOAA-GFDL/fre-workflows.git" - expected_tag = "2025.04" - expected_output = [f"({repo}):({expected_tag}) check out ==> REQUESTED", - f"({repo}):({expected_tag}) check out ==> SUCCESSFUL"] - - for string in expected_output: - assert string in caplog.text - -def test_pp_workflow_checkout_branch_conflict(fake_home, caplog): - """ - Test for the expected ValueError if the checkout was done already, - but user is checking out same repo again, with a different branch/tag. - """ - # 1st checkout: using default 'main' branch as set in the settings.yaml (version) - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - branch = None) - - expected_output = (f"ERROR: Checkout exists ('{fake_home}/cylc-src/{EXPERIMENT}') " - "and does not match '2025.04'") - expected_error = "Neither tag nor branch matches the git clone branch arg" - with pytest.raises(ValueError, match = expected_error): - # 2nd checkout: trying to check out 2025.04 tag while 'main' already checked out - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - branch = "2025.04") - assert expected_output in caplog.text +#def test_pp_workflow_checkout_branch_override(caplog): +# """ +# Test for correct checkout if a '-b', '--branch' is specified. +# """ +# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", +# experiment = EXPERIMENT, +# application = "pp", +# branch = "2025.04") +# +# repo = "https://github.com/NOAA-GFDL/fre-workflows.git" +# expected_tag = "2025.04" +# expected_output = [f"({repo}):({expected_tag}) check out ==> REQUESTED", +# f"({repo}):({expected_tag}) check out ==> SUCCESSFUL"] +# +# for string in expected_output: +# assert string in caplog.text +# +#def test_pp_workflow_checkout_branch_conflict(fake_home, caplog): +# """ +# Test for the expected ValueError if the checkout was done already, +# but user is checking out same repo again, with a different branch/tag. +# """ +# # 1st checkout: using default 'main' branch as set in the settings.yaml (version) +# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", +# experiment = EXPERIMENT, +# application = "pp", +# branch = None) +# +# expected_output = (f"ERROR: Checkout exists ('{fake_home}/cylc-src/{EXPERIMENT}') " +# "and does not match '2025.04'") +# expected_error = "Neither tag nor branch matches the git clone branch arg" +# with pytest.raises(ValueError, match = expected_error): +# # 2nd checkout: trying to check out 2025.04 tag while 'main' already checked out +# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", +# experiment = EXPERIMENT, +# application = "pp", +# branch = "2025.04") +# assert expected_output in caplog.text From 8bef3eeffd6e14852f0fe2680d05bef854e6f9e4 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 11:26:53 -0500 Subject: [PATCH 017/119] #697 Fix file permission test --- fre/workflow/tests/test_checkout_script.py | 25 ++++++++++------------ 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 7f4eb711b..3f19ffa8e 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -28,21 +28,18 @@ def test_cylc_src_creation_fail(fake_home): This test simulates a permission error in HOME. """ - # Temporarily change fake_home permissions to read-only, so cylc-src creation will fail - os.chmod(fake_home, stat.S_IREAD) - try: - if os.geteuid == 0: # if running as root - pytest.skip("Cannot test premission errors when running as root") - print("RUNNING AS ROOT") - AH -# # run checkout to create cylc-src -# directory = Path(f"{fake_home}/cylc-src") -# expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" -# with pytest.raises(OSError, match = re.escape(expected_error)): -# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", -# experiment = EXPERIMENT, -# application = "pp") + # Temporarily change fake_home permissions to read-only for usr/owner + # If read-only, cylc-src dir creation should fail + Path(fake_home).chmod(stat.S_IRUSR) + + # run checkout to create cylc-src + directory = Path(f"{fake_home}/cylc-src") + expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" + with pytest.raises(OSError, match = re.escape(expected_error)): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp") finally: os.chmod(fake_home, stat.S_IRWXU) From 7d1eb4ed86cc81cf30922c34dd1c8c1b1b0cbbd7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 11:27:56 -0500 Subject: [PATCH 018/119] #697 Fix test --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 3f19ffa8e..22b0b158e 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -41,7 +41,7 @@ def test_cylc_src_creation_fail(fake_home): experiment = EXPERIMENT, application = "pp") finally: - os.chmod(fake_home, stat.S_IRWXU) + Path(fake_home).chmod(stat.S_IRWXU) def test_check_missing_repo(): """ From 516cf5d953a100badade1ea4de57ad63688a10e6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 11:47:42 -0500 Subject: [PATCH 019/119] #697 hopfeully change file permissions correctly this time --- fre/workflow/tests/test_checkout_script.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 22b0b158e..bb609aa52 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -31,7 +31,8 @@ def test_cylc_src_creation_fail(fake_home): try: # Temporarily change fake_home permissions to read-only for usr/owner # If read-only, cylc-src dir creation should fail - Path(fake_home).chmod(stat.S_IRUSR) + read_only_mode = 0o44 + Path(fake_home).chmod(read_only_mode) # run checkout to create cylc-src directory = Path(f"{fake_home}/cylc-src") @@ -41,7 +42,8 @@ def test_cylc_src_creation_fail(fake_home): experiment = EXPERIMENT, application = "pp") finally: - Path(fake_home).chmod(stat.S_IRWXU) + rwx_mode = 0o77 + Path(fake_home).chmod(rwx_mode) def test_check_missing_repo(): """ From 95e25d846acb651f6bdc7bc9a98300b30ea7923d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 15:40:02 -0500 Subject: [PATCH 020/119] #697 Nix file permission changes - doesn't work when running as root --- fre/workflow/tests/test_checkout_script.py | 25 +++++++++------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index bb609aa52..76921cc6e 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -28,22 +28,17 @@ def test_cylc_src_creation_fail(fake_home): This test simulates a permission error in HOME. """ - try: - # Temporarily change fake_home permissions to read-only for usr/owner - # If read-only, cylc-src dir creation should fail - read_only_mode = 0o44 - Path(fake_home).chmod(read_only_mode) + cylc_src_file = Path(f"{fake_home}/cylc-src") + with open(cylc_src_file, "w") as f: + f.write("testing 123") - # run checkout to create cylc-src - directory = Path(f"{fake_home}/cylc-src") - expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" - with pytest.raises(OSError, match = re.escape(expected_error)): - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp") - finally: - rwx_mode = 0o77 - Path(fake_home).chmod(rwx_mode) + # run checkout to create cylc-src + directory = Path(f"{fake_home}/cylc-src") + expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" + with pytest.raises(OSError, match = re.escape(expected_error)): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp") def test_check_missing_repo(): """ From e3c2452207e95e218e388b53392fa7dccdda2c37 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 15:56:11 -0500 Subject: [PATCH 021/119] #697 Address pylint messages --- fre/workflow/tests/test_checkout_script.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 76921cc6e..c4a0190b6 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -1,7 +1,5 @@ ''' fre workflow checkout tests ''' -import stat import re -import os from pathlib import Path import pytest from fre.workflow import checkout_script @@ -26,10 +24,11 @@ def test_cylc_src_creation_fail(fake_home): Test for the expected failure if the cylc-src directory cannot be created. - This test simulates a permission error in HOME. + This test simulates a file with the name cylc-src + already created, causing a permission error in HOME. """ cylc_src_file = Path(f"{fake_home}/cylc-src") - with open(cylc_src_file, "w") as f: + with open(cylc_src_file, "w", encoding='utf-8') as f: f.write("testing 123") # run checkout to create cylc-src From 7b1f816204b415eea4b3f31a83f6ff35a301ddda Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 17:33:51 -0500 Subject: [PATCH 022/119] #697 Add fre workflow cli tests --- fre/tests/test_fre_workflow_cli.py | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 fre/tests/test_fre_workflow_cli.py diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py new file mode 100644 index 000000000..def167d2d --- /dev/null +++ b/fre/tests/test_fre_workflow_cli.py @@ -0,0 +1,36 @@ +import os +import shutil +from pathlib import Path + +from click.testing import CliRunner + +from fre import fre + +""" +CLI Tests for fre workflow * +Tests the command-line-interface calls for tools in the fre workflow suite. +Each tool generally gets 3 tests: + - fre workflow $tool, checking for exit code 0 (fails if cli isn't configured right) + - fre workflow $tool --help, checking for exit code 0 (fails if the code doesn't run) + - fre workflow $tool --optionDNE, checking for exit code 2 (fails if cli isn't configured + right and thinks the tool has a --optionDNE option) +""" + +runner = CliRunner() + +#-- fre pp +def test_cli_fre_workflow(): + ''' fre workflow ''' + result = runner.invoke(fre.fre, args=["workflow"]) + assert result.exit_code == 2 + +def test_cli_fre_workflow_help(): + ''' fre workflow --help ''' + result = runner.invoke(fre.fre, args=["workflow", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_workflow_opt_dne(): + ''' fre workflow optionDNE ''' + result = runner.invoke(fre.fre, args=["workflow", "optionDNE"]) + assert result.exit_code == 2 + From c48728b7628e4cb9ef37eb2a3f5d619a5a667dc6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 19 Feb 2026 16:48:11 -0500 Subject: [PATCH 023/119] #697 Some clean up --- fre/workflow/__init__.py | 27 ------------- fre/workflow/checkout_script.py | 1 - fre/workflow/freworkflow.py | 45 ---------------------- fre/workflow/tests/test_checkout_script.py | 39 ------------------- 4 files changed, 112 deletions(-) diff --git a/fre/workflow/__init__.py b/fre/workflow/__init__.py index dc62bb794..e69de29bb 100644 --- a/fre/workflow/__init__.py +++ b/fre/workflow/__init__.py @@ -1,27 +0,0 @@ -from typing import Optional - -def make_workflow_name(experiment : Optional[str] = None) -> str: - """ - Function that takes in a triplet of tags for a model experiment, platform, and target, and - returns a directory name for the corresponding pp workflow. Because this is often given by - user to the shell being used by python, we split/reform the string to remove semi-colons or - spaces that may be used to execute an arbitrary command with elevated privileges. - - :param experiment: One of the postprocessing experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None - :type experiment: str - :param platform: The location + compiler that was used to run the model (e.g. gfdl.ncrc5-deploy), default None - :type platform: str - :param target: Options used for the model compiler (e.g. prod-openmp), default None - :type target: str - :return: string created in specific format from the input strings - :rtype: str - - .. note:: if any arguments are None, then "None" will appear in the workflow name - """ - name = f'{experiment}__{platform}__{target}' - return ''.join( - (''.join( - name.split(' ') - ) - ).split(';') - ) # user-input sanitation, prevents some malicious cmds from being executed with privileges diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 222183ae0..17351aff7 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -7,7 +7,6 @@ import fre.yamltools.combine_yamls_script as cy from fre.app.helpers import change_directory -#from . import make_workflow_name from jsonschema import validate, SchemaError, ValidationError fre_logger = logging.getLogger(__name__) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index ac0964cf4..55d1d15bf 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -29,48 +29,3 @@ def checkout(yamlfile, experiment, application): Checkout/extract fre workflow """ checkout_script.workflow_checkout(yamlfile, experiment, application) - -#@workflow_cli.command() -#@click.option("-e", "--experiment", type=str, -# help="Experiment name", -# required=True) -#def install(experiment): -# """ -# Install workflow configuration -# """ -# install_script.workflow_install(experiment) - -#@workflow_cli.command() -#@click.option("-e", "--experiment", type=str, -# help="Experiment name", -# required=True) -#@click.option("--pause", is_flag=True, default=False, -# help="Pause the workflow immediately on start up", -# required=False) -#@click.option("--no_wait", is_flag=True, default=False, -# help="after submission, do not wait to ping the scheduler and confirm success", -# required=False) -#def run(experiment, pause, no_wait): -# """ -# Run workflow configuration -# """ -# run_script.workflow_run(experiment, pause, no_wait) - -#@workflow_cli.command() -#@click.option("-e", "--experiment", type=str, -# help="Experiment name", -# required=True) -#@click.option("-c", "--config-file", type=str, -# help="Path to a configuration file in either XML or YAML", -# required=True) -#@click.option("-b", "--branch", -# required=False, default=None, -# help="fre-workflows branch/tag to clone; default is $(fre --version)") -#@click.option("-t", "--time", -# required=False, default=None, -# help="Time whose history files are ready") -#def all(experiment, platform, target, config_file, branch, time): -# """ -# Execute all fre workflow initialization steps in order -# """ -# wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index c4a0190b6..5cfc6e189 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -100,42 +100,3 @@ def test_pp_workflow_checkout_exists_already(fake_home, caplog): for string in expected_output: assert string in caplog.text - -#def test_pp_workflow_checkout_branch_override(caplog): -# """ -# Test for correct checkout if a '-b', '--branch' is specified. -# """ -# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", -# experiment = EXPERIMENT, -# application = "pp", -# branch = "2025.04") -# -# repo = "https://github.com/NOAA-GFDL/fre-workflows.git" -# expected_tag = "2025.04" -# expected_output = [f"({repo}):({expected_tag}) check out ==> REQUESTED", -# f"({repo}):({expected_tag}) check out ==> SUCCESSFUL"] -# -# for string in expected_output: -# assert string in caplog.text -# -#def test_pp_workflow_checkout_branch_conflict(fake_home, caplog): -# """ -# Test for the expected ValueError if the checkout was done already, -# but user is checking out same repo again, with a different branch/tag. -# """ -# # 1st checkout: using default 'main' branch as set in the settings.yaml (version) -# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", -# experiment = EXPERIMENT, -# application = "pp", -# branch = None) -# -# expected_output = (f"ERROR: Checkout exists ('{fake_home}/cylc-src/{EXPERIMENT}') " -# "and does not match '2025.04'") -# expected_error = "Neither tag nor branch matches the git clone branch arg" -# with pytest.raises(ValueError, match = expected_error): -# # 2nd checkout: trying to check out 2025.04 tag while 'main' already checked out -# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", -# experiment = EXPERIMENT, -# application = "pp", -# branch = "2025.04") -# assert expected_output in caplog.text From 817bf7a26b08f466d3c2dba7a8b13d42deb27f7b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 14:37:52 -0500 Subject: [PATCH 024/119] #697 Add --force-checkout and --target-dir --- fre/workflow/checkout_script.py | 91 ++++++++++++++++++++++----------- fre/workflow/freworkflow.py | 14 +++-- 2 files changed, 72 insertions(+), 33 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 17351aff7..d9b05ca61 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -3,17 +3,16 @@ import subprocess from pathlib import Path import logging -import shutil +import shutil +import json +from jsonschema import validate, SchemaError, ValidationError +from typing import Optional import fre.yamltools.combine_yamls_script as cy from fre.app.helpers import change_directory -from jsonschema import validate, SchemaError, ValidationError fre_logger = logging.getLogger(__name__) -FRE_PP_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" -FRE_RUN_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" - ######VALIDATE##### def validate_yaml(yamlfile: dict, application: str) -> None: """ @@ -55,7 +54,28 @@ def validate_yaml(yamlfile: dict, application: str) -> None: except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def workflow_checkout(yamlfile: str = None, experiment = None, application = None): +def create_checkout(repo, tag, src_dir, workflow_name): + """ + Create a directory and clone the workflow template files from a defined repo. --------- + ........... + ........... + ........... + ........... + """ + # scenarios 1+2, checkout doesn't exist, branch specified (or not) + clone_output = subprocess.run( ["git", "clone","--recursive", + f"--branch={tag}", + repo, f"{src_dir}/{workflow_name}"], + capture_output = True, text = True, check = True) + fre_logger.debug(clone_output) + fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + + ## Move combined yaml to cylc-src location + current_dir = Path.cwd() + shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") + fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) + +def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a defined repo. @@ -67,13 +87,18 @@ def workflow_checkout(yamlfile: str = None, experiment = None, application = Non :type experiment: str :param application: Which workflow will be used/cloned :type application: str - :raises OSError: why checkout script was not able to be created + :raises OSError: if the checkout script was not able to be created :raises ValueError: - -if experiment or platform or target is None + - if the repo and/or tag was not defined + - if the + raise ValueError('Neither tag nor branch matches the git clone branch arg') + """ # Used in consolidate_yamls function for now platform = None target = None +# print(src_dir) +# quit() if application == "run": fre_logger.info("NOT DONE YET") # will probably be taken out and put above is "use" @@ -94,7 +119,7 @@ def workflow_checkout(yamlfile: str = None, experiment = None, application = Non platform=platform, target=target, use="pp", - output=f"config.yaml") + output="config.yaml") #validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp_workflow") @@ -108,36 +133,42 @@ def workflow_checkout(yamlfile: str = None, experiment = None, application = Non fre_logger.warning("(%s):(%s) check out ==> REQUESTED", repo, tag) + # Make sure src_dir exists + if not Path(target_dir).exists(): + raise ValueError(f"Source directory {target_dir} does not exist or cannot be found.") + # clone directory - directory = os.path.expanduser("~/cylc-src") + src_dir = f"{target_dir}/cylc-src" # workflow name workflow_name = experiment # create workflow in cylc-src try: - Path(directory).mkdir(parents=True, exist_ok=True) + Path(src_dir).mkdir(parents=True, exist_ok=True) except Exception as exc: raise OSError( - f"(checkoutScript) directory {directory} wasn't able to be created. exit!") from exc + f"(checkoutScript) directory {src_dir} wasn't able to be created. exit!") from exc - if not Path(f"{directory}/{workflow_name}").is_dir(): - # scenarios 1+2, checkout doesn't exist, branch specified (or not) + if not Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.info("Workflow does not yet exist; will create now") - clone_output = subprocess.run( ["git", "clone","--recursive", - f"--branch={tag}", - repo, f"{directory}/{workflow_name}"], - capture_output = True, text = True, check = True) - fre_logger.debug(clone_output) - fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) - - ## Move combined yaml to cylc-src location - cylc_src_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{experiment}") - current_dir = Path.cwd() - shutil.move(Path(f"{current_dir}/config.yaml"), cylc_src_dir) - fre_logger.info("Combined yaml file moved to ~/cylc-src/%s", experiment) - else: + create_checkout(repo = repo, + tag = tag, + src_dir = src_dir, + workflow_name = workflow_name) + elif Path(f"{src_dir}/{workflow_name}").is_dir() and force_checkout: + fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) + # Remove checked out repo + shutil.rmtree(f"{src_dir}/{workflow_name}") + fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) + # Redo checkout + create_checkout(repo = repo, + tag = tag, + src_dir = src_dir, + workflow_name = workflow_name) + elif Path(f"{src_dir}/{workflow_name}").is_dir() and not force_checkout: + fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) # the repo checkout does exist, scenarios 3 and 4. - with change_directory(f"{directory}/{workflow_name}"): + with change_directory(f"{src_dir}/{workflow_name}"): # capture the branch and tag # if either match git_clone_branch_arg, then success. otherwise, fail. current_tag = subprocess.run(["git","describe","--tags"], @@ -148,10 +179,10 @@ def workflow_checkout(yamlfile: str = None, experiment = None, application = Non text = True, check = True).stdout.strip() if tag in (current_tag, current_branch): - fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", directory, workflow_name, tag) + fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) else: fre_logger.error( - "ERROR: Checkout exists ('%s/%s') and does not match '%s'", directory, workflow_name, tag) + "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag) fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 55d1d15bf..9360de710 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -1,5 +1,5 @@ ''' fre workflow ''' - +import os import click import logging fre_logger = logging.getLogger(__name__) @@ -24,8 +24,16 @@ def workflow_cli(): type=click.Choice(['run', 'pp']), help="Use case for checked out workflow", required=True) -def checkout(yamlfile, experiment, application): +@click.option("--target-dir", + type=str, + default=lambda: os.environ['TMPDIR'], + help=f"Target directory for workflow to be cloned into. Default location: {os.environ['TMPDIR']}") +@click.option("--force-checkout", + is_flag=True, + default=False, + help="If the checkout already, exists, remove and check out again.") +def checkout(yamlfile, experiment, application, target_dir, force_checkout): """ Checkout/extract fre workflow """ - checkout_script.workflow_checkout(yamlfile, experiment, application) + checkout_script.workflow_checkout(yamlfile, experiment, application, target_dir, force_checkout) From 09afc2db42bd13374df38db1f4ec461922e3e829 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 14:38:28 -0500 Subject: [PATCH 025/119] #697 Update spacing to account for warning output --- fre/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/__init__.py b/fre/__init__.py index cfe401402..9d6b2cef9 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -10,7 +10,7 @@ fre_logger = logging.getLogger(__name__) -FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s" +FORMAT = "[%(levelname)7s:%(filename)24s:%(funcName)24s] %(message)s" logging.basicConfig(level = logging.WARNING, format = FORMAT, filename = None, From a29645afbad7a622668c930db6266cd13bdb010e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 15:20:53 -0500 Subject: [PATCH 026/119] #697 Update documentation --- fre/workflow/checkout_script.py | 33 +++++++++++++++++++++------------ fre/workflow/freworkflow.py | 2 +- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index d9b05ca61..ca16cc0ee 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -22,7 +22,7 @@ def validate_yaml(yamlfile: dict, application: str) -> None: :param yamlfile: Model, settings, pp, and analysis yaml information combined into a dictionary :type yamlfile: dict - :param application: ------------------------------------------------ + :param application: type of workflow to check out/clone :type application: string :raises ValueError: - if gfdl_mdf_schema path is not valid @@ -54,13 +54,19 @@ def validate_yaml(yamlfile: dict, application: str) -> None: except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def create_checkout(repo, tag, src_dir, workflow_name): +def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str) --> None: """ - Create a directory and clone the workflow template files from a defined repo. --------- - ........... - ........... - ........... - ........... + Clone the workflow template files from a defined repo into the cylc-src/workflow + directory and move the resolved yaml to the cylc-src directory. + + :param repo: Yaml defined workflow repository + :type repo: str + :param tag: branch or version of defined repository + :type tag: str + :param src_dir: Cylc-src directory + :type src_dir: str + :param workflow_name: Name of workflow + :type workflow_name: src """ # scenarios 1+2, checkout doesn't exist, branch specified (or not) clone_output = subprocess.run( ["git", "clone","--recursive", @@ -75,7 +81,7 @@ def create_checkout(repo, tag, src_dir, workflow_name): shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) -def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False): +def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False) --> None: """ Create a directory and clone the workflow template files from a defined repo. @@ -87,9 +93,14 @@ def workflow_checkout(yamlfile: str = None, experiment: str = None, application: :type experiment: str :param application: Which workflow will be used/cloned :type application: str + :param target_dir: Target directory to clone repository into + :type target_dir: str + :param force_checkout: re-clone the workflow repo if it exists + :type force_checkout: bool :raises OSError: if the checkout script was not able to be created :raises ValueError: - if the repo and/or tag was not defined + - if the target directory does not exist or cannot be found - if the raise ValueError('Neither tag nor branch matches the git clone branch arg') @@ -97,8 +108,7 @@ def workflow_checkout(yamlfile: str = None, experiment: str = None, application: # Used in consolidate_yamls function for now platform = None target = None -# print(src_dir) -# quit() + if application == "run": fre_logger.info("NOT DONE YET") # will probably be taken out and put above is "use" @@ -124,7 +134,6 @@ def workflow_checkout(yamlfile: str = None, experiment: str = None, application: workflow_info = yaml.get("workflow").get("pp_workflow") repo = workflow_info.get("repo") - tag = workflow_info.get("version") fre_logger.info("Defined tag ==> '%s'", tag) @@ -135,7 +144,7 @@ def workflow_checkout(yamlfile: str = None, experiment: str = None, application: # Make sure src_dir exists if not Path(target_dir).exists(): - raise ValueError(f"Source directory {target_dir} does not exist or cannot be found.") + raise ValueError(f"Target directory {target_dir} does not exist or cannot be found.") # clone directory src_dir = f"{target_dir}/cylc-src" diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 9360de710..06573c927 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -22,7 +22,7 @@ def workflow_cli(): required=True) @click.option("-a", "--application", type=click.Choice(['run', 'pp']), - help="Use case for checked out workflow", + help="Type of workflow to check out/clone", required=True) @click.option("--target-dir", type=str, From 560093cf8e27931af877659319cfe0a07a649d3b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 15:32:13 -0500 Subject: [PATCH 027/119] #697 Update doc --- fre/workflow/checkout_script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index ca16cc0ee..e04a080fe 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -14,7 +14,7 @@ fre_logger = logging.getLogger(__name__) ######VALIDATE##### -def validate_yaml(yamlfile: dict, application: str) -> None: +def validate_yaml(yamlfile: dict, application: str): """ Validate the format of the yaml file based on the schema.json in gfdl_msd_schemas @@ -54,7 +54,7 @@ def validate_yaml(yamlfile: dict, application: str) -> None: except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str) --> None: +def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str): """ Clone the workflow template files from a defined repo into the cylc-src/workflow directory and move the resolved yaml to the cylc-src directory. @@ -81,7 +81,7 @@ def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str) --> N shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) -def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False) --> None: +def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a defined repo. From 10ce0091eb7f6eb01c8d82445beee3651d6af2b2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 16:32:59 -0500 Subject: [PATCH 028/119] #697 Fix tests --- fre/workflow/checkout_script.py | 2 +- fre/workflow/tests/test_checkout_script.py | 77 ++++++++++++++-------- 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index e04a080fe..e368574a3 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -81,7 +81,7 @@ def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str): shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) -def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False): +def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a defined repo. diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 5cfc6e189..a21c7b5ab 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -7,19 +7,19 @@ TEST_CONFIGS = "fre/workflow/tests/AM5_example/" EXPERIMENT = "c96L65_am5f7b12r1_amip_TESTING" -@pytest.fixture(autouse=True, name="fake_home") -def fake_home_fixture(tmp_path, monkeypatch): - """ - Set the tmp_path as HOME for the cylc-src directory - to be created in. - """ - ## Mock HOME for cylc-src and cylc-run - fake_home = Path(tmp_path) - monkeypatch.setenv("HOME", str(fake_home)) - - return fake_home - -def test_cylc_src_creation_fail(fake_home): +#@pytest.fixture(autouse=True, name="fake_home") +#def fake_home_fixture(tmp_path, monkeypatch): +# """ +# Set the tmp_path as HOME for the cylc-src directory +# to be created in. +# """ +# ## Mock HOME for cylc-src and cylc-run +# fake_home = Path(tmp_path) +# monkeypatch.setenv("HOME", str(fake_home)) +# +# return fake_home +# +def test_cylc_src_creation_fail(tmp_path): """ Test for the expected failure if the cylc-src directory cannot be created. @@ -27,19 +27,34 @@ def test_cylc_src_creation_fail(fake_home): This test simulates a file with the name cylc-src already created, causing a permission error in HOME. """ - cylc_src_file = Path(f"{fake_home}/cylc-src") + cylc_src_file = Path(f"{tmp_path}/cylc-src") with open(cylc_src_file, "w", encoding='utf-8') as f: f.write("testing 123") # run checkout to create cylc-src - directory = Path(f"{fake_home}/cylc-src") + directory = Path(f"{tmp_path}/cylc-src") expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" with pytest.raises(OSError, match = re.escape(expected_error)): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp") + application = "pp", + target_dir = tmp_path) -def test_check_missing_repo(): +def test_checkout_target_dir_dne(tmp_path): + """ + """ + bad_dir = "does/not/exist" + + # run checkout to create cylc-src + expected_error = f"Target directory {bad_dir} does not exist or cannot be found." + + with pytest.raises(ValueError, match = re.escape(expected_error)): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = bad_dir) + +def test_checkout_missing_repo(tmp_path): """ Test for the expected ValueError if the repo is not defined in the settings.yaml. @@ -50,7 +65,8 @@ def test_check_missing_repo(): with pytest.raises(ValueError, match = f"One of these are None: repo / tag = {repo} / {tag}"): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = experiment, - application = "pp") + application = "pp", + target_dir = tmp_path) #def test_run_workflow_checkout(caplog): # """ @@ -60,43 +76,48 @@ def test_check_missing_repo(): # experiment = "c96L65_am5f7b12r1_amip_TESTING", # application = "run") -def test_pp_workflow_checkout(fake_home, caplog): +def test_pp_workflow_checkout(tmp_path, caplog): """ Test for a successful post-processing workflow checkout. """ checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp") + application = "pp", + target_dir = tmp_path) expected_repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" - assert all([Path(f"{fake_home}/cylc-src/{EXPERIMENT}").exists(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}").is_dir(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists(), + assert all([Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").exists(), + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").is_dir(), + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/config.yaml").exists(), f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) -def test_pp_workflow_checkout_exists_already(fake_home, caplog): +def test_pp_workflow_checkout_exists_already(tmp_path, caplog): """ Test for the expected output message if the checkout/branch already exists. """ # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp") + application = "pp", + target_dir = tmp_path) # 2nd checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp") + application = "pp", + target_dir = tmp_path) repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" expected_output = [ f"({repo}):({expected_tag}) check out ==> REQUESTED", - (f"Checkout exists ('{fake_home}/cylc-src/c96L65_am5f7b12r1_amip_TESTING'), " + (f"Checkout exists ('{tmp_path}/cylc-src/c96L65_am5f7b12r1_amip_TESTING'), " f"and matches '{expected_tag}'") ] for string in expected_output: assert string in caplog.text + +##def test_pp_workflow_checkout_force_checkout From 61be15c9253526a8b2923c5e788c9930753cc301 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 16:53:51 -0500 Subject: [PATCH 029/119] #697 Fix --target-dir click value - Use TMPDIR env variable if command line option not passed --- fre/workflow/freworkflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 06573c927..d45dc773d 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -26,8 +26,8 @@ def workflow_cli(): required=True) @click.option("--target-dir", type=str, - default=lambda: os.environ['TMPDIR'], - help=f"Target directory for workflow to be cloned into. Default location: {os.environ['TMPDIR']}") + envvar="TMPDIR", + help=f"Target directory for workflow to be cloned into. Default location TMPDIR: {os.environ['TMPDIR']}") @click.option("--force-checkout", is_flag=True, default=False, From 04e8985ef8615f96059127640c7a03e4b7cd6761 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 17:11:39 -0500 Subject: [PATCH 030/119] #697 Add default dir (see if pipeline passes) --- fre/workflow/freworkflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index d45dc773d..61caf3d45 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -27,6 +27,7 @@ def workflow_cli(): @click.option("--target-dir", type=str, envvar="TMPDIR", + default=".fre" help=f"Target directory for workflow to be cloned into. Default location TMPDIR: {os.environ['TMPDIR']}") @click.option("--force-checkout", is_flag=True, From 9daa391a0b0e2e368d63a9a9ffb43cefdf794b98 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 17:42:30 -0500 Subject: [PATCH 031/119] #697 update --- fre/workflow/checkout_script.py | 4 ++-- fre/workflow/freworkflow.py | 2 +- fre/workflow/tests/AM5_example/yaml_include/settings.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index e368574a3..35af24d7e 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -120,7 +120,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N use="run", output=None) #validate_yaml(yamlfile = yaml, application = "run") - workflow_info = yaml.get("workflow").get("run_workflow") + workflow_info = yaml.get("workflow").get("run") elif application == "pp": # will probably be taken out and put above is "use" # is generalized in this tool @@ -131,7 +131,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N use="pp", output="config.yaml") #validate_yaml(yamlfile = yaml, application = "pp") - workflow_info = yaml.get("workflow").get("pp_workflow") + workflow_info = yaml.get("workflow").get("pp") repo = workflow_info.get("repo") tag = workflow_info.get("version") diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 61caf3d45..9b5f87bd1 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -27,7 +27,7 @@ def workflow_cli(): @click.option("--target-dir", type=str, envvar="TMPDIR", - default=".fre" + default=".fre", help=f"Target directory for workflow to be cloned into. Default location TMPDIR: {os.environ['TMPDIR']}") @click.option("--force-checkout", is_flag=True, diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml index 791c0c437..c45d9c533 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -1,9 +1,9 @@ #workflow repositories workflow: - run_workflow: + run: repo: "https://github.com/NOAA-GFDL" version: "tbd" - pp_workflow: + pp: repo: "https://github.com/NOAA-GFDL/fre-workflows.git" version: "main" From 14e87e9be90786f772016d0dde76a522ec88d880 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 17:57:03 -0500 Subject: [PATCH 032/119] #697 update --- .../tests/AM5_example/yaml_include/settings_WRONG.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml index c973591e8..312a5e7cd 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -1,9 +1,9 @@ #workflow repositories workflow: - run_workflow: + run: repo: "https://github.com/NOAA-GFDL" version: "tbd" - pp_workflow: + pp: repo: version: "main" From 7af0425b8954850e891ce958fee1ecb71e4cd647 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 12:51:05 -0500 Subject: [PATCH 033/119] #697 Fix tests and update argument order --- fre/tests/test_fre_workflow_cli.py | 101 +++++++++++++++++---- fre/workflow/checkout_script.py | 18 ++-- fre/workflow/freworkflow.py | 10 +- fre/workflow/tests/test_checkout_script.py | 78 ++++++++-------- 4 files changed, 136 insertions(+), 71 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index def167d2d..bd9b1c595 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -1,36 +1,103 @@ -import os -import shutil -from pathlib import Path - -from click.testing import CliRunner - -from fre import fre - """ CLI Tests for fre workflow * Tests the command-line-interface calls for tools in the fre workflow suite. + Each tool generally gets 3 tests: - - fre workflow $tool, checking for exit code 0 (fails if cli isn't configured right) - - fre workflow $tool --help, checking for exit code 0 (fails if the code doesn't run) - - fre workflow $tool --optionDNE, checking for exit code 2 (fails if cli isn't configured - right and thinks the tool has a --optionDNE option) + - fre workflow $tool, checking for exit code 0 (fails if cli isn't configured right) + - fre workflow $tool --help, checking for exit code 0 (fails if the code doesn't run) + - fre workflow $tool --optionDNE, checking for exit code 2; misuse of command (fails if cli isn't configured + right and thinks the tool has a --optionDNE option) """ +import os +from pathlib import Path +from click.testing import CliRunner +from fre import fre runner = CliRunner() -#-- fre pp -def test_cli_fre_workflow(): +## fre workflow subtools search for if TMPDIR is set, specifically for fre workflow checkout --target-dir +# If a value for --target-dir is not passed --> TMPDIR will be used for --target-dir +# If TMPDIR is not set --> a default location will be used for --target-dir +#-- fre workflow +def test_cli_fre_workflow(monkeypatch): ''' fre workflow ''' + monkeypatch.setenv("TMPDIR", "") result = runner.invoke(fre.fre, args=["workflow"]) - assert result.exit_code == 2 + assert result.exit_code == 0 -def test_cli_fre_workflow_help(): +def test_cli_fre_workflow_help(monkeypatch): ''' fre workflow --help ''' + monkeypatch.setenv("TMPDIR", "") result = runner.invoke(fre.fre, args=["workflow", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_opt_dne(): +def test_cli_fre_workflow_opt_dne(monkeypatch): ''' fre workflow optionDNE ''' + monkeypatch.setenv("TMPDIR", "") result = runner.invoke(fre.fre, args=["workflow", "optionDNE"]) assert result.exit_code == 2 +#-- fre workflow checkout +def test_cli_fre_workflow_checkout(monkeypatch): + ''' fre workflow checkout''' + monkeypatch.setenv("TMPDIR", "") + result = runner.invoke(fre.fre, args=["workflow", "checkout"]) + assert result.exit_code == 2 + +def test_cli_fre_workflow_checkout_help(monkeypatch): + ''' fre workflow checkout --help ''' + monkeypatch.setenv("TMPDIR", "") + result = runner.invoke(fre.fre, args=["workflow", "checkout", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_workflow_checkout_opt_dne(monkeypatch): + ''' fre workflow checkout optionDNE ''' + monkeypatch.setenv("TMPDIR", "") + result = runner.invoke(fre.fre, args=["workflow", "checkout", "optionDNE"]) + assert result.exit_code == 2 + +def test_cli_fre_workflow_checkout_target_dir_set(tmp_path): + """ + Test checkout in target directory if --target-dir is explicitly set. + """ + experiment = "c96L65_am5f7b12r1_amip_TESTING" + result = runner.invoke(fre.fre, args=["workflow", "checkout", + "--yamlfile", "fre/workflow/tests/AM5_example/am5.yaml", + "--experiment", experiment, + "--application", "pp", + "--target-dir", tmp_path]) + assert result.exit_code == 0 + assert Path(f"{tmp_path}/cylc-src/{experiment}").exists() + +def test_cli_fre_workflow_checkout_TMPDIR_set(tmp_path, monkeypatch): + """ + Test checkout if TMPDIR environment variable is set and --target-dir has no + specified value + """ + experiment = "c96L65_am5f7b12r1_amip_TESTING" + Path(f"{tmp_path}/env_var").mkdir(parents=True) + monkeypatch.setenv("TMPDIR", f"{tmp_path}/env_var") + + result = runner.invoke(fre.fre, args=["workflow", "checkout", + "-y", "fre/workflow/tests/AM5_example/am5.yaml", + "-e", experiment, + "-a", "pp"]) + assert result.exit_code == 0 + assert Path(f"{os.environ['TMPDIR']}/cylc-src/{experiment}").exists() + +#def test_cli_fre_workflow_checkout_default_dir(): +# """ +# Test checkout if TMPDIR and --target-dir is not set; +# use the default location: ~/.fre +# """ + experiment = "c96L65_am5f7b12r1_amip_TESTING" + monkeypatch.setenv("TMPDIR", "") + + result = runner.invoke(fre.fre, args=["workflow", "checkout", + "-y", "fre/workflow/tests/AM5_example/am5.yaml", + "-e", experiment, + "-a", "pp"]) + #default cylc-src location + default_dir = os.path.expanduser("~/.fre") + assert result.exit_code == 0 + assert Path(f"{default_dir}/cylc-src/{experiment}").exists() diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 35af24d7e..ffdf49812 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -28,8 +28,6 @@ def validate_yaml(yamlfile: dict, application: str): - if gfdl_mdf_schema path is not valid - combined yaml is not valid - unclear error in validation - :return: None - :rtype: None """ schema_dir = Path(__file__).resolve().parents[1] schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json') @@ -74,7 +72,7 @@ def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str): repo, f"{src_dir}/{workflow_name}"], capture_output = True, text = True, check = True) fre_logger.debug(clone_output) - fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + fre_logger.info("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) ## Move combined yaml to cylc-src location current_dir = Path.cwd() @@ -93,7 +91,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N :type experiment: str :param application: Which workflow will be used/cloned :type application: str - :param target_dir: Target directory to clone repository into + :param target_dir: Target/base directory used for cylc-src/ creation :type target_dir: str :param force_checkout: re-clone the workflow repo if it exists :type force_checkout: bool @@ -101,9 +99,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N :raises ValueError: - if the repo and/or tag was not defined - if the target directory does not exist or cannot be found - - if the - raise ValueError('Neither tag nor branch matches the git clone branch arg') - + - if neither tag nor branch matches the git clone branch arg """ # Used in consolidate_yamls function for now platform = None @@ -119,7 +115,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use="run", output=None) - #validate_yaml(yamlfile = yaml, application = "run") + validate_yaml(yamlfile = yaml, application = "run") workflow_info = yaml.get("workflow").get("run") elif application == "pp": # will probably be taken out and put above is "use" @@ -142,11 +138,11 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N fre_logger.warning("(%s):(%s) check out ==> REQUESTED", repo, tag) - # Make sure src_dir exists + # Create src_dir if it does not exist if not Path(target_dir).exists(): - raise ValueError(f"Target directory {target_dir} does not exist or cannot be found.") + Path(target_dir).mkdir(parents=True, exist_ok=True) - # clone directory + # Define cylc-src directory src_dir = f"{target_dir}/cylc-src" # workflow name workflow_name = experiment diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 9b5f87bd1..b31f5e98e 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -27,14 +27,14 @@ def workflow_cli(): @click.option("--target-dir", type=str, envvar="TMPDIR", - default=".fre", - help=f"Target directory for workflow to be cloned into. Default location TMPDIR: {os.environ['TMPDIR']}") + default=os.path.expanduser("~/.fre"), + help=f"Target directory for workflow to be cloned into. TMPDIR will be used if set: {os.environ['TMPDIR']}. If not set, a default location of ") @click.option("--force-checkout", is_flag=True, default=False, - help="If the checkout already, exists, remove and check out again.") -def checkout(yamlfile, experiment, application, target_dir, force_checkout): + help="If the checkout already, exists, remove and clone the desired repo again.") +def checkout(target_dir, yamlfile, experiment, application, force_checkout): """ Checkout/extract fre workflow """ - checkout_script.workflow_checkout(yamlfile, experiment, application, target_dir, force_checkout) + checkout_script.workflow_checkout(target_dir, yamlfile, experiment, application, force_checkout) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index a21c7b5ab..f30b06779 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -7,18 +7,6 @@ TEST_CONFIGS = "fre/workflow/tests/AM5_example/" EXPERIMENT = "c96L65_am5f7b12r1_amip_TESTING" -#@pytest.fixture(autouse=True, name="fake_home") -#def fake_home_fixture(tmp_path, monkeypatch): -# """ -# Set the tmp_path as HOME for the cylc-src directory -# to be created in. -# """ -# ## Mock HOME for cylc-src and cylc-run -# fake_home = Path(tmp_path) -# monkeypatch.setenv("HOME", str(fake_home)) -# -# return fake_home -# def test_cylc_src_creation_fail(tmp_path): """ Test for the expected failure if the cylc-src @@ -40,20 +28,6 @@ def test_cylc_src_creation_fail(tmp_path): application = "pp", target_dir = tmp_path) -def test_checkout_target_dir_dne(tmp_path): - """ - """ - bad_dir = "does/not/exist" - - # run checkout to create cylc-src - expected_error = f"Target directory {bad_dir} does not exist or cannot be found." - - with pytest.raises(ValueError, match = re.escape(expected_error)): - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - target_dir = bad_dir) - def test_checkout_missing_repo(tmp_path): """ Test for the expected ValueError if the repo is not @@ -68,14 +42,6 @@ def test_checkout_missing_repo(tmp_path): application = "pp", target_dir = tmp_path) -#def test_run_workflow_checkout(caplog): -# """ -# Test for a successful run workflow checkout. -# """ -# checkout_script.workflow_checkout(yamlfile, -# experiment = "c96L65_am5f7b12r1_amip_TESTING", -# application = "run") - def test_pp_workflow_checkout(tmp_path, caplog): """ Test for a successful post-processing workflow checkout. @@ -90,12 +56,13 @@ def test_pp_workflow_checkout(tmp_path, caplog): assert all([Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").exists(), Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").is_dir(), Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), - Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/config.yaml").exists(), - f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/config.yaml").exists()]) +# f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) def test_pp_workflow_checkout_exists_already(tmp_path, caplog): """ - Test for the expected output message if the checkout/branch already exists. + Test for the expected output message if the checkout already exists, + using the same branch. """ # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", @@ -120,4 +87,39 @@ def test_pp_workflow_checkout_exists_already(tmp_path, caplog): for string in expected_output: assert string in caplog.text -##def test_pp_workflow_checkout_force_checkout +def test_pp_workflow_checkout_force_checkout(tmp_path, caplog): + """ + Test successful re-cloning of the workflow repo if + force-checkout is passed. + """ + # 1st checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path) + + # 2nd checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path, + force_checkout = True) + + src_dir = f"{tmp_path}/cylc-src" + workflow_name = EXPERIMENT + repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + tag = "main" + expected_output = [f" *** PREVIOUS CHECKOUT FOUND: {src_dir}/{workflow_name} *** ", + f" *** REMOVING {src_dir}/{workflow_name} *** "] + # f"({repo}):({tag}) check out ==> SUCCESSFUL"] + + for string in expected_output: + assert string in caplog.text + +#def test_run_workflow_checkout(caplog): +# """ +# Test for a successful run workflow checkout. +# """ +# checkout_script.workflow_checkout(yamlfile, +# experiment = "c96L65_am5f7b12r1_amip_TESTING", +# application = "run") From 5ee7f19c55ed9eb2d657704cddea243829bb5c4f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 14:01:47 -0500 Subject: [PATCH 034/119] #697 fix exit code --- fre/tests/test_fre_workflow_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index bd9b1c595..54033fcf5 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -23,7 +23,7 @@ def test_cli_fre_workflow(monkeypatch): ''' fre workflow ''' monkeypatch.setenv("TMPDIR", "") result = runner.invoke(fre.fre, args=["workflow"]) - assert result.exit_code == 0 + assert result.exit_code == 2 def test_cli_fre_workflow_help(monkeypatch): ''' fre workflow --help ''' From 007696d73cd7f34624f4b236f48e4ffa44667f79 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 14:48:45 -0500 Subject: [PATCH 035/119] #697 Adjust spacing --- fre/tests/test_fre_cmor_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index f273a4e48..4e8fb180f 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -70,8 +70,8 @@ def test_cli_fre_cmor_help_and_debuglog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long - log_text_line_2='[DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_2='[ DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] @@ -89,7 +89,7 @@ def test_cli_fre_cmor_help_and_infolog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] From 9ab07ca6063425bbf82f0bddc0302bedf321da1f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 15:03:07 -0500 Subject: [PATCH 036/119] #697 right spacing? --- fre/tests/test_fre_cmor_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index 4e8fb180f..82263e5a7 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -70,8 +70,8 @@ def test_cli_fre_cmor_help_and_debuglog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long - log_text_line_2='[ DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_2='[ DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] @@ -89,7 +89,7 @@ def test_cli_fre_cmor_help_and_infolog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] From 2d26bceb8ec6145892873565adea452279abc0a2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 15:23:28 -0500 Subject: [PATCH 037/119] #697 set TMPDIR --- fre/tests/test_fre_workflow_cli.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index 54033fcf5..479ad20e1 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -19,40 +19,40 @@ # If a value for --target-dir is not passed --> TMPDIR will be used for --target-dir # If TMPDIR is not set --> a default location will be used for --target-dir #-- fre workflow -def test_cli_fre_workflow(monkeypatch): +def test_cli_fre_workflow(monkeypatch, tmp_path): ''' fre workflow ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow"]) assert result.exit_code == 2 -def test_cli_fre_workflow_help(monkeypatch): +def test_cli_fre_workflow_help(monkeypatch, tmp_path): ''' fre workflow --help ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_opt_dne(monkeypatch): +def test_cli_fre_workflow_opt_dne(monkeypatch, tmp_path): ''' fre workflow optionDNE ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "optionDNE"]) assert result.exit_code == 2 #-- fre workflow checkout -def test_cli_fre_workflow_checkout(monkeypatch): +def test_cli_fre_workflow_checkout(monkeypatch, tmp_path): ''' fre workflow checkout''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout"]) assert result.exit_code == 2 -def test_cli_fre_workflow_checkout_help(monkeypatch): +def test_cli_fre_workflow_checkout_help(monkeypatch, tmp_path): ''' fre workflow checkout --help ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_checkout_opt_dne(monkeypatch): +def test_cli_fre_workflow_checkout_opt_dne(monkeypatch, tmp_path): ''' fre workflow checkout optionDNE ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout", "optionDNE"]) assert result.exit_code == 2 From 1af099201bc7f4afb7e89b48ed5426b5ca4b31da Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 15:24:09 -0500 Subject: [PATCH 038/119] #697 Change warning to info --- fre/workflow/checkout_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index ffdf49812..15b24654e 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -126,7 +126,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use="pp", output="config.yaml") - #validate_yaml(yamlfile = yaml, application = "pp") + validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp") repo = workflow_info.get("repo") @@ -136,7 +136,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N if None in [repo, tag]: raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}") - fre_logger.warning("(%s):(%s) check out ==> REQUESTED", repo, tag) + fre_logger.info("(%s):(%s) check out ==> REQUESTED", repo, tag) # Create src_dir if it does not exist if not Path(target_dir).exists(): From 14c0d20f3560c37c108ea70db1e995b011ed43ba Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 15:53:42 -0500 Subject: [PATCH 039/119] #697 Update help message and other output --- fre/workflow/checkout_script.py | 4 ++-- fre/workflow/freworkflow.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 15b24654e..caf8f0c27 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -44,7 +44,7 @@ def validate_yaml(yamlfile: dict, application: str): # If the yaml is not valid, the schema validation will raise errors and exit try: validate(instance = yamlfile,schema=schema) - fre_logger.info("Combined yaml valid") + fre_logger.info(" ** COMBINED YAML VALID ** ") except SchemaError as exc: raise ValueError(f"Schema '{schema_path}' is not valid. Contact the FRE team.") from exc except ValidationError as exc: @@ -126,7 +126,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use="pp", output="config.yaml") - validate_yaml(yamlfile = yaml, application = "pp") + #validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp") repo = workflow_info.get("repo") diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index b31f5e98e..0e07bda1b 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -28,7 +28,10 @@ def workflow_cli(): type=str, envvar="TMPDIR", default=os.path.expanduser("~/.fre"), - help=f"Target directory for workflow to be cloned into. TMPDIR will be used if set: {os.environ['TMPDIR']}. If not set, a default location of ") + help=f"""Target directory for the workflow to be cloned into. + If not defined, the environment variable TMPDIR will be used, + if available. If both --target-dir and TMPDIR are not set, a + default location of ~/.fre will be used""") @click.option("--force-checkout", is_flag=True, default=False, From ef0377b77276f407c6dc15a164ddffba1680dfd8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 16:14:41 -0500 Subject: [PATCH 040/119] #697 Uncomment validation --- fre/workflow/checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index caf8f0c27..437f5410b 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -126,7 +126,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use="pp", output="config.yaml") - #validate_yaml(yamlfile = yaml, application = "pp") + validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp") repo = workflow_info.get("repo") From 779b6ba2b08280b02da93ea756354bdd65208b29 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Feb 2026 14:06:27 -0500 Subject: [PATCH 041/119] #697 Address ians comments --- fre/workflow/checkout_script.py | 118 +++++++-------------- fre/workflow/tests/test_checkout_script.py | 3 - 2 files changed, 39 insertions(+), 82 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 437f5410b..8c4ce1534 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -52,33 +52,6 @@ def validate_yaml(yamlfile: dict, application: str): except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str): - """ - Clone the workflow template files from a defined repo into the cylc-src/workflow - directory and move the resolved yaml to the cylc-src directory. - - :param repo: Yaml defined workflow repository - :type repo: str - :param tag: branch or version of defined repository - :type tag: str - :param src_dir: Cylc-src directory - :type src_dir: str - :param workflow_name: Name of workflow - :type workflow_name: src - """ - # scenarios 1+2, checkout doesn't exist, branch specified (or not) - clone_output = subprocess.run( ["git", "clone","--recursive", - f"--branch={tag}", - repo, f"{src_dir}/{workflow_name}"], - capture_output = True, text = True, check = True) - fre_logger.debug(clone_output) - fre_logger.info("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) - - ## Move combined yaml to cylc-src location - current_dir = Path.cwd() - shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") - fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) - def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a defined repo. @@ -105,29 +78,16 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N platform = None target = None - if application == "run": - fre_logger.info("NOT DONE YET") - # will probably be taken out and put above is "use" - # is generalized in this tool + if application in ["run", "pp"]: + fre_logger.info(" ** Configuring the resolved YAML for the %s **", application) yaml = cy.consolidate_yamls(yamlfile=yamlfile, experiment=experiment, platform=platform, target=target, - use="run", - output=None) - validate_yaml(yamlfile = yaml, application = "run") - workflow_info = yaml.get("workflow").get("run") - elif application == "pp": - # will probably be taken out and put above is "use" - # is generalized in this tool - yaml = cy.consolidate_yamls(yamlfile=yamlfile, - experiment=experiment, - platform=platform, - target=target, - use="pp", + use=application, output="config.yaml") - validate_yaml(yamlfile = yaml, application = "pp") - workflow_info = yaml.get("workflow").get("pp") + validate_yaml(yamlfile = yaml, application = application) + workflow_info = yaml.get("workflow").get(application) repo = workflow_info.get("repo") tag = workflow_info.get("version") @@ -154,40 +114,40 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N raise OSError( f"(checkoutScript) directory {src_dir} wasn't able to be created. exit!") from exc - if not Path(f"{src_dir}/{workflow_name}").is_dir(): - fre_logger.info("Workflow does not yet exist; will create now") - create_checkout(repo = repo, - tag = tag, - src_dir = src_dir, - workflow_name = workflow_name) - elif Path(f"{src_dir}/{workflow_name}").is_dir() and force_checkout: + if Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) - # Remove checked out repo - shutil.rmtree(f"{src_dir}/{workflow_name}") - fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) - # Redo checkout - create_checkout(repo = repo, - tag = tag, - src_dir = src_dir, - workflow_name = workflow_name) - elif Path(f"{src_dir}/{workflow_name}").is_dir() and not force_checkout: - fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) - # the repo checkout does exist, scenarios 3 and 4. - with change_directory(f"{src_dir}/{workflow_name}"): - # capture the branch and tag - # if either match git_clone_branch_arg, then success. otherwise, fail. - current_tag = subprocess.run(["git","describe","--tags"], - capture_output = True, - text = True, check = True).stdout.strip() - current_branch = subprocess.run(["git", "branch", "--show-current"], + if force_checkout: + fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) + shutil.rmtree(f"{src_dir}/{workflow_name}") + else: + with change_directory(f"{src_dir}/{workflow_name}"): + # capture the branch and tag + # if either match git_clone_branch_arg, then success. otherwise, fail. + current_tag = subprocess.run(["git","describe","--tags"], capture_output = True, text = True, check = True).stdout.strip() - - if tag in (current_tag, current_branch): - fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) - else: - fre_logger.error( - "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag) - fre_logger.error( - "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) - raise ValueError('Neither tag nor branch matches the git clone branch arg') + current_branch = subprocess.run(["git", "branch", "--show-current"], + capture_output = True, + text = True, check = True).stdout.strip() + + if tag in (current_tag, current_branch): + fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) + else: + fre_logger.error( + "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag) + fre_logger.error( + "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) + raise ValueError('Neither tag nor branch matches the git clone branch arg') + else: + fre_logger.info("Workflow does not exist; will create now") + clone_output = subprocess.run( ["git", "clone","--recursive", + f"--branch={tag}", + repo, f"{src_dir}/{workflow_name}"], + capture_output = True, text = True, check = True) + fre_logger.debug(clone_output) + fre_logger.info("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + + ## Move combined yaml to cylc-src location + current_dir = Path.cwd() + shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") + fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index f30b06779..8456cf97b 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -57,7 +57,6 @@ def test_pp_workflow_checkout(tmp_path, caplog): Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").is_dir(), Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/config.yaml").exists()]) -# f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) def test_pp_workflow_checkout_exists_already(tmp_path, caplog): """ @@ -79,7 +78,6 @@ def test_pp_workflow_checkout_exists_already(tmp_path, caplog): repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" expected_output = [ - f"({repo}):({expected_tag}) check out ==> REQUESTED", (f"Checkout exists ('{tmp_path}/cylc-src/c96L65_am5f7b12r1_amip_TESTING'), " f"and matches '{expected_tag}'") ] @@ -111,7 +109,6 @@ def test_pp_workflow_checkout_force_checkout(tmp_path, caplog): tag = "main" expected_output = [f" *** PREVIOUS CHECKOUT FOUND: {src_dir}/{workflow_name} *** ", f" *** REMOVING {src_dir}/{workflow_name} *** "] - # f"({repo}):({tag}) check out ==> SUCCESSFUL"] for string in expected_output: assert string in caplog.text From 6ce4930e9122c5aa03017f85e3037219b2750fb7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Feb 2026 15:12:19 -0500 Subject: [PATCH 042/119] #697 Update --- fre/workflow/checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 8c4ce1534..8b9f797e0 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -138,7 +138,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') - else: + if not Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.info("Workflow does not exist; will create now") clone_output = subprocess.run( ["git", "clone","--recursive", f"--branch={tag}", From 0bd5f6008847d94d8b2167ff0728da41763e734a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 5 Mar 2026 13:51:11 -0500 Subject: [PATCH 043/119] #697 Update yaml workflow addition - remove duplication of "pp" and "run" - put workflow repo and version under associated section that already exists --- fre/workflow/checkout_script.py | 10 ++++++++-- .../tests/AM5_example/yaml_include/settings.yaml | 12 +++--------- .../AM5_example/yaml_include/settings_WRONG.yaml | 12 +++--------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 8b9f797e0..cd84917b1 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -86,8 +86,14 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use=application, output="config.yaml") + validate_yaml(yamlfile = yaml, application = application) - workflow_info = yaml.get("workflow").get(application) + + # Reset application for pp to make it discoverable in yaml config + if application == "pp": + application = "postprocess" + + workflow_info = yaml.get(application).get("workflow") repo = workflow_info.get("repo") tag = workflow_info.get("version") @@ -96,7 +102,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N if None in [repo, tag]: raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}") - fre_logger.info("(%s):(%s) check out ==> REQUESTED", repo, tag) + fre_logger.info("(%s):(%s) check out for %s ==> REQUESTED", repo, tag, application) # Create src_dir if it does not exist if not Path(target_dir).exists(): diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml index c45d9c533..4bf3770c4 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -1,12 +1,3 @@ -#workflow repositories -workflow: - run: - repo: "https://github.com/NOAA-GFDL" - version: "tbd" - pp: - repo: "https://github.com/NOAA-GFDL/fre-workflows.git" - version: "main" - #c96_amip_directories: directories: history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] @@ -16,6 +7,9 @@ directories: #c96_amip_postprocess: postprocess: + workflow: + repo: "https://github.com/NOAA-GFDL/fre-workflows.git" + version: "main" settings: history_segment: "P1Y" site: "ppan" diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml index 312a5e7cd..9a102b685 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -1,12 +1,3 @@ -#workflow repositories -workflow: - run: - repo: "https://github.com/NOAA-GFDL" - version: "tbd" - pp: - repo: - version: "main" - #c96_amip_directories: directories: history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] @@ -16,6 +7,9 @@ directories: #c96_amip_postprocess: postprocess: + workflow: + repo: + version: "main" settings: history_segment: "P1Y" site: "ppan" From e542b60ba1f364e3bfaf2a63c965c88283aabb64 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 5 Mar 2026 14:01:10 -0500 Subject: [PATCH 044/119] #697 Update readme --- fre/workflow/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 1006fc61e..19915872a 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -15,3 +15,5 @@ fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7 - `-y, --yamlfile [model yaml] (str; required)` - `-e, --experiment [experiment name] (str; required)` - `-a, --application [ run | pp ] (str; required)` + - `--target-dir [target directory where workflow will be cloned] (str; optional; default - $TMPDIR if set; default if TMPDIR not set - ~/.fre` + - `--force-checkout (bool; optional)` From b1f46644d266148082b0981a6d94f735e0a06db2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 12:58:05 -0400 Subject: [PATCH 045/119] #697 Some documentation updates --- fre/workflow/checkout_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index cd84917b1..2b8787a1d 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -17,7 +17,7 @@ def validate_yaml(yamlfile: dict, application: str): """ Validate the format of the yaml file based - on the schema.json in gfdl_msd_schemas + on the schema.json held in [gfdl_msd_schemas](https://github.com/NOAA-GFDL/gfdl_msd_schemas). :param yamlfile: Model, settings, pp, and analysis yaml information combined into a dictionary @@ -54,7 +54,7 @@ def validate_yaml(yamlfile: dict, application: str): def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): """ - Create a directory and clone the workflow template files from a defined repo. + Create a directory and clone the workflow template files from a defined repository. :param yamlfile: Model yaml configuration file :type yamlfile: str From fd672d3f024105dcc48ad2e402b0aa1046b52791 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 14:12:21 -0400 Subject: [PATCH 046/119] #697 change repo to repository --- fre/workflow/tests/AM5_example/yaml_include/settings.yaml | 2 +- fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml index 4bf3770c4..44e2ab797 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -8,7 +8,7 @@ directories: #c96_amip_postprocess: postprocess: workflow: - repo: "https://github.com/NOAA-GFDL/fre-workflows.git" + repository: "https://github.com/NOAA-GFDL/fre-workflows.git" version: "main" settings: history_segment: "P1Y" diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml index 9a102b685..538be7d28 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -8,7 +8,7 @@ directories: #c96_amip_postprocess: postprocess: workflow: - repo: + repository: version: "main" settings: history_segment: "P1Y" From 98fafcbb4baa12775c4e4a1046540df5292971d8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 14:22:13 -0400 Subject: [PATCH 047/119] #697 change repo to repository --- fre/workflow/checkout_script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 2b8787a1d..ab6b5cac5 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -66,11 +66,11 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N :type application: str :param target_dir: Target/base directory used for cylc-src/ creation :type target_dir: str - :param force_checkout: re-clone the workflow repo if it exists + :param force_checkout: re-clone the workflow repository if it exists :type force_checkout: bool :raises OSError: if the checkout script was not able to be created :raises ValueError: - - if the repo and/or tag was not defined + - if the repository and/or tag was not defined - if the target directory does not exist or cannot be found - if neither tag nor branch matches the git clone branch arg """ @@ -95,7 +95,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N workflow_info = yaml.get(application).get("workflow") - repo = workflow_info.get("repo") + repo = workflow_info.get("repository") tag = workflow_info.get("version") fre_logger.info("Defined tag ==> '%s'", tag) From 7ad2290ba9e96ad06665e4826ef61c3bf87d28fa Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 14:26:02 -0400 Subject: [PATCH 048/119] #697 update test --- .../tests/AM5_example/yaml_include/settings_WRONG.yaml | 2 +- fre/workflow/tests/test_checkout_script.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml index 538be7d28..f620f8028 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -8,7 +8,7 @@ directories: #c96_amip_postprocess: postprocess: workflow: - repository: + repository: "" version: "main" settings: history_segment: "P1Y" diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 8456cf97b..b69f20bad 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -28,15 +28,16 @@ def test_cylc_src_creation_fail(tmp_path): application = "pp", target_dir = tmp_path) -def test_checkout_missing_repo(tmp_path): +def test_checkout_invalid_resolved_yaml(tmp_path): """ - Test for the expected ValueError if the repo is not - defined in the settings.yaml. + Test for the expected error if the repository is not + defined in the settings.yaml and the yamls could not + be combined """ experiment = "c96L65_am5f7b12r1_amip_TESTING_WRONG" repo = None tag = "main" - with pytest.raises(ValueError, match = f"One of these are None: repo / tag = {repo} / {tag}"): + with pytest.raises(ValueError, match = f"Combined yaml is not valid. Please fix the errors and try again."): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = experiment, application = "pp", From f62089fd7fbee40531a63aea0e9f128c2d1322d5 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 15:18:59 -0400 Subject: [PATCH 049/119] #697 Change `fre_logger.warning` to `fre_logger.info` --- fre/workflow/checkout_script.py | 2 +- fre/workflow/tests/test_checkout_script.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index ab6b5cac5..a9218aaba 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -121,7 +121,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N f"(checkoutScript) directory {src_dir} wasn't able to be created. exit!") from exc if Path(f"{src_dir}/{workflow_name}").is_dir(): - fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) + fre_logger.info(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) if force_checkout: fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) shutil.rmtree(f"{src_dir}/{workflow_name}") diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index b69f20bad..5a4cf07d9 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -108,11 +108,9 @@ def test_pp_workflow_checkout_force_checkout(tmp_path, caplog): workflow_name = EXPERIMENT repo = "https://github.com/NOAA-GFDL/fre-workflows.git" tag = "main" - expected_output = [f" *** PREVIOUS CHECKOUT FOUND: {src_dir}/{workflow_name} *** ", - f" *** REMOVING {src_dir}/{workflow_name} *** "] + expected_output = f" *** REMOVING {src_dir}/{workflow_name} *** " - for string in expected_output: - assert string in caplog.text + assert expected_output in caplog.text #def test_run_workflow_checkout(caplog): # """ From 361f65fa003ec794c3671d3f8a8115a1b9844b42 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 17:05:32 -0400 Subject: [PATCH 050/119] #697 Remove envvar TMPDIR usage as default for now --- fre/workflow/README.md | 2 +- fre/workflow/freworkflow.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 19915872a..c0759c2a5 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -15,5 +15,5 @@ fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7 - `-y, --yamlfile [model yaml] (str; required)` - `-e, --experiment [experiment name] (str; required)` - `-a, --application [ run | pp ] (str; required)` - - `--target-dir [target directory where workflow will be cloned] (str; optional; default - $TMPDIR if set; default if TMPDIR not set - ~/.fre` + - `--target-dir [target directory where workflow will be cloned] (str; optional; default is ~/.fre-workflows` - `--force-checkout (bool; optional)` diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 0e07bda1b..be8c6a4ea 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -26,12 +26,10 @@ def workflow_cli(): required=True) @click.option("--target-dir", type=str, - envvar="TMPDIR", - default=os.path.expanduser("~/.fre"), + default=os.path.expanduser("~/.fre-workflows"), help=f"""Target directory for the workflow to be cloned into. - If not defined, the environment variable TMPDIR will be used, - if available. If both --target-dir and TMPDIR are not set, a - default location of ~/.fre will be used""") + If not defined, a default location of ~/.fre-workflows + will be used""") @click.option("--force-checkout", is_flag=True, default=False, From a5e931434c4b093179b7d929047fb3eeabdb2b14 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 17:07:33 -0400 Subject: [PATCH 051/119] #697 update doc --- fre/workflow/freworkflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index be8c6a4ea..29bdfc97e 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -1,4 +1,4 @@ -''' fre workflow ''' +''' fre workflow click interface for fre workflow subcommands''' import os import click import logging From d59b3b60b6428fa5b1e91086c2e58a122073d131 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 16 Mar 2026 11:45:50 -0400 Subject: [PATCH 052/119] #697 Address some of Mikyung's comments --- fre/workflow/checkout_script.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index a9218aaba..95ceb6917 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -16,17 +16,17 @@ ######VALIDATE##### def validate_yaml(yamlfile: dict, application: str): """ - Validate the format of the yaml file based - on the schema.json held in [gfdl_msd_schemas](https://github.com/NOAA-GFDL/gfdl_msd_schemas). + Validate the format of the yaml file against the + schema.json held in [gfdl_msd_schemas](https://github.com/NOAA-GFDL/gfdl_msd_schemas). - :param yamlfile: Model, settings, pp, and analysis yaml - information combined into a dictionary + :param yamlfile: Dictionary containing the combined model, + settings, pp, and analysis yaml content :type yamlfile: dict :param application: type of workflow to check out/clone :type application: string :raises ValueError: - - if gfdl_mdf_schema path is not valid - - combined yaml is not valid + - invalid gfdl_msd_schema path + - invalid combined yaml - unclear error in validation """ schema_dir = Path(__file__).resolve().parents[1] @@ -54,25 +54,26 @@ def validate_yaml(yamlfile: dict, application: str): def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): """ - Create a directory and clone the workflow template files from a defined repository. + Create a directory and clone the workflow template files from a specified repository. :param yamlfile: Model yaml configuration file :type yamlfile: str - :param experiment: One of the postprocessing experiment names from the - yaml displayed by fre list exps -y $yamlfile - (e.g. c96L65_am5f4b4r0_amip), default None + :param experiment: One of the experiment names listed in the model yaml file. + Note: the command "fre list exps -y [model_yamlfile]" can be used to + list the available experiment names :type experiment: str - :param application: Which workflow will be used/cloned + :param application: String used to specify the type of workflow to be used/cloned. + Ex.: run, postprocess :type application: str - :param target_dir: Target/base directory used for cylc-src/ creation + :param target_dir: Target location to create the cylc-src/ directory in :type target_dir: str :param force_checkout: re-clone the workflow repository if it exists :type force_checkout: bool - :raises OSError: if the checkout script was not able to be created + :raises OSError: if the checkout script cannot be created :raises ValueError: - if the repository and/or tag was not defined - if the target directory does not exist or cannot be found - - if neither tag nor branch matches the git clone branch arg + - if tag or branch does not match the git clone branch arg """ # Used in consolidate_yamls function for now platform = None From 2411f6ea56494ccc54116746644cc6fa1e9aa421 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 16 Mar 2026 11:57:13 -0400 Subject: [PATCH 053/119] #697 addressing comments pt. 2 --- fre/tests/test_fre_workflow_cli.py | 13 ++++--------- fre/workflow/tests/test_checkout_script.py | 8 ++++---- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index 479ad20e1..b2845c910 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -1,12 +1,10 @@ """ CLI Tests for fre workflow * -Tests the command-line-interface calls for tools in the fre workflow suite. -Each tool generally gets 3 tests: - - fre workflow $tool, checking for exit code 0 (fails if cli isn't configured right) - - fre workflow $tool --help, checking for exit code 0 (fails if the code doesn't run) - - fre workflow $tool --optionDNE, checking for exit code 2; misuse of command (fails if cli isn't configured - right and thinks the tool has a --optionDNE option) +Tests the command-line-interface commands for each tool in the fre workflow suite. + - successful invocation of fre workflow $tool + - successful invocation of fre workflow $tool --help + - expected failure for fre workflow $tool --optionDne (failure for no click option defined for what was passed) """ import os from pathlib import Path @@ -15,9 +13,6 @@ runner = CliRunner() -## fre workflow subtools search for if TMPDIR is set, specifically for fre workflow checkout --target-dir -# If a value for --target-dir is not passed --> TMPDIR will be used for --target-dir -# If TMPDIR is not set --> a default location will be used for --target-dir #-- fre workflow def test_cli_fre_workflow(monkeypatch, tmp_path): ''' fre workflow ''' diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 5a4cf07d9..135ae1aaa 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -12,8 +12,8 @@ def test_cylc_src_creation_fail(tmp_path): Test for the expected failure if the cylc-src directory cannot be created. - This test simulates a file with the name cylc-src - already created, causing a permission error in HOME. + This test simulates the instance where a file with the name + 'cylc-src' already exists, causing a permission error in HOME. """ cylc_src_file = Path(f"{tmp_path}/cylc-src") with open(cylc_src_file, "w", encoding='utf-8') as f: @@ -88,8 +88,8 @@ def test_pp_workflow_checkout_exists_already(tmp_path, caplog): def test_pp_workflow_checkout_force_checkout(tmp_path, caplog): """ - Test successful re-cloning of the workflow repo if - force-checkout is passed. + Test successful re-cloning of the workflow repo + when force-checkout=True. """ # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", From 37bc15856aec5febd6941d4d97c07bb4c3b5075c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 16 Mar 2026 17:58:06 -0400 Subject: [PATCH 054/119] #697 address comments pt 3 --- fre/tests/test_fre_workflow_cli.py | 2 +- fre/workflow/tests/test_checkout_script.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index b2845c910..90559f515 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -4,7 +4,7 @@ Tests the command-line-interface commands for each tool in the fre workflow suite. - successful invocation of fre workflow $tool - successful invocation of fre workflow $tool --help - - expected failure for fre workflow $tool --optionDne (failure for no click option defined for what was passed) + - expected failure for fre workflow $tool --optionDne (failure for undefined click option) """ import os from pathlib import Path diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 135ae1aaa..34d0a39b0 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -31,7 +31,7 @@ def test_cylc_src_creation_fail(tmp_path): def test_checkout_invalid_resolved_yaml(tmp_path): """ Test for the expected error if the repository is not - defined in the settings.yaml and the yamls could not + defined in the settings.yaml and the yamls cannot be combined """ experiment = "c96L65_am5f7b12r1_amip_TESTING_WRONG" @@ -61,8 +61,7 @@ def test_pp_workflow_checkout(tmp_path, caplog): def test_pp_workflow_checkout_exists_already(tmp_path, caplog): """ - Test for the expected output message if the checkout already exists, - using the same branch. + Test for the expected output message if the checkout already exists. """ # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", From b6f8d6541cc30fe50683fbef9b35e04b5c56f01b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 19 Mar 2026 15:09:58 -0400 Subject: [PATCH 055/119] #697 update doc --- fre/workflow/freworkflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 29bdfc97e..11cace332 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -36,6 +36,6 @@ def workflow_cli(): help="If the checkout already, exists, remove and clone the desired repo again.") def checkout(target_dir, yamlfile, experiment, application, force_checkout): """ - Checkout/extract fre workflow + Checkout/clone the workflow repository. """ checkout_script.workflow_checkout(target_dir, yamlfile, experiment, application, force_checkout) From e1459efd82d0d880b699113b4e250681ae59c87f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 14:20:53 -0400 Subject: [PATCH 056/119] #697 fix wording --- fre/workflow/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index c0759c2a5..270f85862 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -1,19 +1,20 @@ # FRE workflow -`fre workflow` provides subtools that help to clone, install, and run a workflow from a repository. + +The`fre workflow` toolset allows user to clone, install, and run a cylc workflow. ## Quickstart -From the root of the fre-cli repository, run: +From the top-level dircetory of the fre-cli repository: ``` # Checkout/clone the post-processing workflow repository -fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING -a pp +fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING --application pp ``` ## Subtools - `fre workflow checkout [options]` - - Purpose: Clone the workflow repository/branch, depending on the application passed. + - Purpose: Clone the specified workflow repository from the settings.yaml, associated with the application passed. - Options: - `-y, --yamlfile [model yaml] (str; required)` - `-e, --experiment [experiment name] (str; required)` - `-a, --application [ run | pp ] (str; required)` - - `--target-dir [target directory where workflow will be cloned] (str; optional; default is ~/.fre-workflows` + - `--target-dir [target location where workflow will be cloned] (str; optional; default is ~/.fre-workflows` - `--force-checkout (bool; optional)` From cf09e3d5027192662c35ec308c5c432fb873c2a6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 16:08:25 -0400 Subject: [PATCH 057/119] #697 Fix tests - define target_dir default in script, not in argument/click option --- fre/tests/test_fre_workflow_cli.py | 50 +++++++++--------------------- fre/workflow/freworkflow.py | 1 - 2 files changed, 14 insertions(+), 37 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index 90559f515..db2a002ae 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -6,7 +6,6 @@ - successful invocation of fre workflow $tool --help - expected failure for fre workflow $tool --optionDne (failure for undefined click option) """ -import os from pathlib import Path from click.testing import CliRunner from fre import fre @@ -14,40 +13,34 @@ runner = CliRunner() #-- fre workflow -def test_cli_fre_workflow(monkeypatch, tmp_path): +def test_cli_fre_workflow(): ''' fre workflow ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow"]) assert result.exit_code == 2 -def test_cli_fre_workflow_help(monkeypatch, tmp_path): +def test_cli_fre_workflow_help(): ''' fre workflow --help ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_opt_dne(monkeypatch, tmp_path): +def test_cli_fre_workflow_opt_dne(): ''' fre workflow optionDNE ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "optionDNE"]) assert result.exit_code == 2 #-- fre workflow checkout -def test_cli_fre_workflow_checkout(monkeypatch, tmp_path): +def test_cli_fre_workflow_checkout(): ''' fre workflow checkout''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout"]) assert result.exit_code == 2 -def test_cli_fre_workflow_checkout_help(monkeypatch, tmp_path): +def test_cli_fre_workflow_checkout_help(): ''' fre workflow checkout --help ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_checkout_opt_dne(monkeypatch, tmp_path): +def test_cli_fre_workflow_checkout_opt_dne(): ''' fre workflow checkout optionDNE ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout", "optionDNE"]) assert result.exit_code == 2 @@ -64,35 +57,20 @@ def test_cli_fre_workflow_checkout_target_dir_set(tmp_path): assert result.exit_code == 0 assert Path(f"{tmp_path}/cylc-src/{experiment}").exists() -def test_cli_fre_workflow_checkout_TMPDIR_set(tmp_path, monkeypatch): +def test_cli_fre_workflow_checkout_default_dir(monkeypatch, tmp_path): """ - Test checkout if TMPDIR environment variable is set and --target-dir has no - specified value + Test workflow repository is cloned in the default location + if --target-dir is not set; default = ~./fre-workflows """ - experiment = "c96L65_am5f7b12r1_amip_TESTING" - Path(f"{tmp_path}/env_var").mkdir(parents=True) - monkeypatch.setenv("TMPDIR", f"{tmp_path}/env_var") - - result = runner.invoke(fre.fre, args=["workflow", "checkout", - "-y", "fre/workflow/tests/AM5_example/am5.yaml", - "-e", experiment, - "-a", "pp"]) - assert result.exit_code == 0 - assert Path(f"{os.environ['TMPDIR']}/cylc-src/{experiment}").exists() + # Create and set a mock HOME + fake_home = f"{tmp_path}/fake_home" + Path(fake_home).mkdir(parents=True,exist_ok=True) + monkeypatch.setenv("HOME", f"{tmp_path}/fake_home") -#def test_cli_fre_workflow_checkout_default_dir(): -# """ -# Test checkout if TMPDIR and --target-dir is not set; -# use the default location: ~/.fre -# """ experiment = "c96L65_am5f7b12r1_amip_TESTING" - monkeypatch.setenv("TMPDIR", "") - result = runner.invoke(fre.fre, args=["workflow", "checkout", "-y", "fre/workflow/tests/AM5_example/am5.yaml", "-e", experiment, "-a", "pp"]) - #default cylc-src location - default_dir = os.path.expanduser("~/.fre") assert result.exit_code == 0 - assert Path(f"{default_dir}/cylc-src/{experiment}").exists() + assert Path(f"{fake_home}/.fre-workflows/cylc-src/{experiment}").exists() diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 11cace332..e2b485031 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -26,7 +26,6 @@ def workflow_cli(): required=True) @click.option("--target-dir", type=str, - default=os.path.expanduser("~/.fre-workflows"), help=f"""Target directory for the workflow to be cloned into. If not defined, a default location of ~/.fre-workflows will be used""") From ce50e1f64add7a0ce98a207f77ce34a16cab9954 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 16:33:42 -0400 Subject: [PATCH 058/119] #697 archive the workflow directory instead, address comments --- fre/workflow/checkout_script.py | 58 +++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 95ceb6917..6c60767c3 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -1,12 +1,14 @@ """ Workflow checkout """ import os import subprocess +import filecmp from pathlib import Path import logging import shutil +from datetime import datetime +from typing import Optional import json from jsonschema import validate, SchemaError, ValidationError -from typing import Optional import fre.yamltools.combine_yamls_script as cy from fre.app.helpers import change_directory @@ -27,7 +29,7 @@ def validate_yaml(yamlfile: dict, application: str): :raises ValueError: - invalid gfdl_msd_schema path - invalid combined yaml - - unclear error in validation + - miscellaneous error in validation """ schema_dir = Path(__file__).resolve().parents[1] schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json') @@ -50,9 +52,10 @@ def validate_yaml(yamlfile: dict, application: str): except ValidationError as exc: raise ValueError("Combined yaml is not valid. Please fix the errors and try again.") from exc except Exception as exc: - raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc + raise ValueError("Miscellaneous error from validation. Please try to find the error and try again.") from exc -def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): +def workflow_checkout(target_dir: str = None, yamlfile: str = None, experiment: str = None, + application: str = None, force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a specified repository. @@ -79,6 +82,10 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N platform = None target = None + # Set the default target directory location + if target_dir is None: + target_dir = os.path.expanduser("~/.fre-workflows") + if application in ["run", "pp"]: fre_logger.info(" ** Configuring the resolved YAML for the %s **", application) yaml = cy.consolidate_yamls(yamlfile=yamlfile, @@ -96,6 +103,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N workflow_info = yaml.get(application).get("workflow") + yaml_filepath = f"{Path.cwd()}/config.yaml" repo = workflow_info.get("repository") tag = workflow_info.get("version") fre_logger.info("Defined tag ==> '%s'", tag) @@ -124,10 +132,24 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N if Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.info(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) if force_checkout: - fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) - shutil.rmtree(f"{src_dir}/{workflow_name}") + # Create archived workflows location + archived = f"{target_dir}/archived_workflows" + Path(archived).mkdir(parents=True, exist_ok=True) + + # Move previous workflow to archived location + fre_logger.warning(" *** Moving previous checkout to %s ***", archived) + shutil.move(f"{src_dir}/{workflow_name}", archived) + + # Rename previous workflow + move_timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + os.rename(f"{archived}/{workflow_name}", f"{archived}/{workflow_name}_{move_timestamp}") + +# # Keeping this here in case we want to switch force_checkout to removing instead of moving +# fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) +# shutil.rmtree(f"{src_dir}/{workflow_name}") else: with change_directory(f"{src_dir}/{workflow_name}"): + ## Compare previous workflow directory # capture the branch and tag # if either match git_clone_branch_arg, then success. otherwise, fail. current_tag = subprocess.run(["git","describe","--tags"], @@ -138,13 +160,35 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N text = True, check = True).stdout.strip() if tag in (current_tag, current_branch): - fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) + fre_logger.info("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) else: fre_logger.error( "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag) fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') + + + ## Compare content of current and previous configured, resolved yamls + if filecmp.cmp(yaml_filepath, "config.yaml", shallow=False): + fre_logger.info("Resolved yaml already exists and did not change.") + else: + fre_logger.error("") + fre_logger.error("ERROR: Checkout and resolved yaml already exist but resolved yaml files " + "are not identical!") + fre_logger.error("For troubleshooting:") + fre_logger.error(" - Current resolved yaml: %s", yaml_filepath) + fre_logger.error(" - Previous resolved yaml: %s", f"{src_dir}/{workflow_name}/config.yaml") + fre_logger.error("Try:") + fre_logger.error(" - resolving yaml differences if nothing else has changed") + fre_logger.error(f" - removing the {target_dir}/cylc-src/{workflow_name} folder and " + "re-running the command") + fre_logger.error(" - pass the --force-checkout option to archive the workflow (move to " + "~/.fre-workflows/archived/) and clone a new workflow.") + return +# raise ValueError("Resolve yaml differences or pass --force-checkout to archive the workflow" +# "(moved to ~/.fre-workflows/archived) and clone a new workflow.") + if not Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.info("Workflow does not exist; will create now") clone_output = subprocess.run( ["git", "clone","--recursive", From c2886ca6a1fcdb01687c1f3b8fb0709f9708a728 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 16:51:42 -0400 Subject: [PATCH 059/119] #697 update doc --- fre/workflow/checkout_script.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 6c60767c3..6d025f8bc 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -70,7 +70,8 @@ def workflow_checkout(target_dir: str = None, yamlfile: str = None, experiment: :type application: str :param target_dir: Target location to create the cylc-src/ directory in :type target_dir: str - :param force_checkout: re-clone the workflow repository if it exists + :param force_checkout: If the workflow directory exists, move it to an archived location + (~/.fre-workflows/archived) and re-clone the workflow repository :type force_checkout: bool :raises OSError: if the checkout script cannot be created :raises ValueError: From 2164715a0a7b9aff8f026f349f754d5e5d168a2f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 16:59:20 -0400 Subject: [PATCH 060/119] #697 update doc --- fre/workflow/README.md | 4 +++- fre/workflow/checkout_script.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 270f85862..75bc72dc3 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -1,6 +1,8 @@ # FRE workflow -The`fre workflow` toolset allows user to clone, install, and run a cylc workflow. +The`fre workflow` toolset allows users to clone, install, and run a cylc workflow. + +The workflow repository and version are specified in `the setting.yaml`. ## Quickstart From the top-level dircetory of the fre-cli repository: diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 6d025f8bc..744c4bc0f 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -65,7 +65,7 @@ def workflow_checkout(target_dir: str = None, yamlfile: str = None, experiment: Note: the command "fre list exps -y [model_yamlfile]" can be used to list the available experiment names :type experiment: str - :param application: String used to specify the type of workflow to be used/cloned. + :param application: String used to specify the type of workflow to be cloned. Ex.: run, postprocess :type application: str :param target_dir: Target location to create the cylc-src/ directory in From 6923876fc42b59ce2633116cb6dbd43a7b8b355e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 27 Jan 2026 15:42:14 -0500 Subject: [PATCH 061/119] #697 Add fre workflow checkout tool and test --- fre/fre.py | 3 +- fre/workflow/README.md | 0 fre/workflow/__init__.py | 27 ++++ fre/workflow/checkout_script.py | 120 ++++++++++++++ fre/workflow/freworkflow.py | 101 ++++++++++++ fre/workflow/tests/AM5_example/am5.yaml | 71 +++++++++ .../yaml_include/pp-test.c96_amip.yaml | 33 ++++ .../AM5_example/yaml_include/pp.c96_amip.yaml | 94 +++++++++++ .../AM5_example/yaml_include/settings.yaml | 32 ++++ .../yaml_include/settings_WRONG.yaml | 32 ++++ fre/workflow/tests/test_checkout_script.py | 147 ++++++++++++++++++ 11 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 fre/workflow/README.md create mode 100644 fre/workflow/__init__.py create mode 100644 fre/workflow/checkout_script.py create mode 100644 fre/workflow/freworkflow.py create mode 100644 fre/workflow/tests/AM5_example/am5.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/settings.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml create mode 100644 fre/workflow/tests/test_checkout_script.py diff --git a/fre/fre.py b/fre/fre.py index 25b717504..2c60308d0 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -22,7 +22,8 @@ # click and lazy group loading @click.group( cls = LazyGroup, - lazy_subcommands = {"pp": ".pp.frepp.pp_cli", + lazy_subcommands = {"workflow": ".workflow.freworkflow.workflow_cli", + "pp": ".pp.frepp.pp_cli", "catalog": ".catalog.frecatalog.catalog_cli", "list": ".list_.frelist.list_cli", "check": ".check.frecheck.check_cli", diff --git a/fre/workflow/README.md b/fre/workflow/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/fre/workflow/__init__.py b/fre/workflow/__init__.py new file mode 100644 index 000000000..dc62bb794 --- /dev/null +++ b/fre/workflow/__init__.py @@ -0,0 +1,27 @@ +from typing import Optional + +def make_workflow_name(experiment : Optional[str] = None) -> str: + """ + Function that takes in a triplet of tags for a model experiment, platform, and target, and + returns a directory name for the corresponding pp workflow. Because this is often given by + user to the shell being used by python, we split/reform the string to remove semi-colons or + spaces that may be used to execute an arbitrary command with elevated privileges. + + :param experiment: One of the postprocessing experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None + :type experiment: str + :param platform: The location + compiler that was used to run the model (e.g. gfdl.ncrc5-deploy), default None + :type platform: str + :param target: Options used for the model compiler (e.g. prod-openmp), default None + :type target: str + :return: string created in specific format from the input strings + :rtype: str + + .. note:: if any arguments are None, then "None" will appear in the workflow name + """ + name = f'{experiment}__{platform}__{target}' + return ''.join( + (''.join( + name.split(' ') + ) + ).split(';') + ) # user-input sanitation, prevents some malicious cmds from being executed with privileges diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py new file mode 100644 index 000000000..58539b1fd --- /dev/null +++ b/fre/workflow/checkout_script.py @@ -0,0 +1,120 @@ +""" Workflow checkout """ +import os +import subprocess +from pathlib import Path +import logging + +import fre.yamltools.combine_yamls_script as cy +from fre.app.helpers import change_directory +#from . import make_workflow_name + +fre_logger = logging.getLogger(__name__) + +FRE_PP_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" +FRE_RUN_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" + +def workflow_checkout(yamlfile: str = None, experiment = None, + application = None, branch = None): + """ + Create a directory and clone the workflow template files from a defined repo. + + :param yamlfile: Model yaml configuration file + :type yamlfile: str + :param experiment: One of the postprocessing experiment names from the + yaml displayed by fre list exps -y $yamlfile + (e.g. c96L65_am5f4b4r0_amip), default None + :type experiment: str + :param platform: The location + compiler that was used to run the model + (e.g. gfdl.ncrc5-deploy), default None + :type platform: str + :param target: Options used for the model compiler (e.g. prod-openmp), default None + :type target: str + :param branch: which git branch to pull from, default None + :type branch: str + :param application: Which workflow will be used/cloned + :type application: str + :raises OSError: why checkout script was not able to be created + :raises ValueError: + -if experiment or platform or target is None + -if branch argument cannot be found as a branch or tag + """ + # Used in consolidate_yamls function for now + platform = None + target = None + if application == "run": + fre_logger.info("NOT DONE YET") + # will probably be taken out and put above is "use" + # is generalized in this tool + yaml = cy.consolidate_yamls(yamlfile=yamlfile, + experiment=experiment, + platform=platform, + target=target, + use="run", + output=None) + workflow_info = yaml.get("workflow").get("run_workflow") + elif application == "pp": + # will probably be taken out and put above is "use" + # is generalized in this tool + yaml = cy.consolidate_yamls(yamlfile=yamlfile, + experiment=experiment, + platform=platform, + target=target, + use="pp", + output=None) + workflow_info = yaml.get("workflow").get("pp_workflow") + + repo = workflow_info.get("repo") + + if not branch: + tag = workflow_info.get("version") + fre_logger.info("Default tag ==> '%s'", tag) + else: + tag = branch + fre_logger.info("Requested branch/tag ==> '%s'", tag) + + if None in [repo, tag]: + raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}") + + fre_logger.warning("(%s):(%s) check out ==> REQUESTED", repo, tag) + + # clone directory + directory = os.path.expanduser("~/cylc-src") + # workflow name + workflow_name = experiment + + # create workflow in cylc-src + try: + Path(directory).mkdir(parents=True, exist_ok=True) + except Exception as exc: + raise OSError( + f"(checkoutScript) directory {directory} wasn't able to be created. exit!") from exc + + if not Path(f"{directory}/{workflow_name}").is_dir(): + # scenarios 1+2, checkout doesn't exist, branch specified (or not) + fre_logger.info("Workflow does not yet exist; will create now") + clone_output = subprocess.run( ["git", "clone","--recursive", + f"--branch={tag}", + repo, f"{directory}/{workflow_name}"], + capture_output = True, text = True, check = True) + fre_logger.debug(clone_output) + fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + else: + # the repo checkout does exist, scenarios 3 and 4. + with change_directory(f"{directory}/{workflow_name}"): + # capture the branch and tag + # if either match git_clone_branch_arg, then success. otherwise, fail. + current_tag = subprocess.run(["git","describe","--tags"], + capture_output = True, + text = True, check = True).stdout.strip() + current_branch = subprocess.run(["git", "branch", "--show-current"], + capture_output = True, + text = True, check = True).stdout.strip() + + if tag in (current_tag, current_branch): + fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", directory, workflow_name, tag) + else: + fre_logger.error( + "ERROR: Checkout exists ('%s/%s') and does not match '%s'", directory, workflow_name, tag) + fre_logger.error( + "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) + raise ValueError('Neither tag nor branch matches the git clone branch arg') diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py new file mode 100644 index 000000000..21b6fec44 --- /dev/null +++ b/fre/workflow/freworkflow.py @@ -0,0 +1,101 @@ +''' fre workflow ''' + +import click +import logging +fre_logger = logging.getLogger(__name__) + +#fre tools +from . import checkout_script +from . import install_script +from . import run_script + +@click.group(help=click.style(" - workflow subcommands", fg=(57,139,210))) +def workflow_cli(): + ''' entry point to fre workflow click commands ''' + +@workflow_cli.command() +@click.option("-y", "--yamlfile", type=str, + help="Model yaml file", + required=True) +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +#@click.option("-p", "--platform", type=str, +# help="Platform name") +#@click.option("-t", "--target", type=str, +# help="Target name") +@click.option("-b", "--branch", type =str, + required=False, default = None, + help="fre-workflows branch/tag to clone; default is $(fre --version)") +@click.option("-a", "--application", + type=click.Choice(['run', 'pp']), + help="Use case for checked out workflow", + required=True) +def checkout(yamlfile, experiment, application, branch=None): + """ + Checkout/extract fre workflow + """ + checkout_script.workflow_checkout(yamlfile, experiment, application, branch) + +@workflow_cli.command() +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +#@click.option("-p", "--platform", type=str, +# help="Platform name", +# required=True) +#@click.option("-t", "--target", type=str, +# help="Target name", +# required=True) +def install(experiment): + """ + Install workflow configuration + """ + install_script.workflow_install(experiment) + +@workflow_cli.command() +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +#@click.option("-p", "--platform", type=str, +# help="Platform name", +# required=True) +#@click.option("-t", "--target", type=str, +# help="Target name", +# required=True) +@click.option("--pause", is_flag=True, default=False, + help="Pause the workflow immediately on start up", + required=False) +@click.option("--no_wait", is_flag=True, default=False, + help="after submission, do not wait to ping the scheduler and confirm success", + required=False) +def run(experiment, pause, no_wait): + """ + Run workflow configuration + """ + run_script.workflow_run(experiment, pause, no_wait) + +@workflow_cli.command() +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +#@click.option("-p", "--platform", type=str, +# help="Platform name", +# required=True) +#@click.option("-T", "--target", type=str, +# help="Target name", +# required=True) +@click.option("-c", "--config-file", type=str, + help="Path to a configuration file in either XML or YAML", + required=True) +@click.option("-b", "--branch", + required=False, default=None, + help="fre-workflows branch/tag to clone; default is $(fre --version)") +@click.option("-t", "--time", + required=False, default=None, + help="Time whose history files are ready") +def all(experiment, platform, target, config_file, branch, time): + """ + Execute all fre workflow initialization steps in order + """ + wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) diff --git a/fre/workflow/tests/AM5_example/am5.yaml b/fre/workflow/tests/AM5_example/am5.yaml new file mode 100644 index 000000000..f40d51a0a --- /dev/null +++ b/fre/workflow/tests/AM5_example/am5.yaml @@ -0,0 +1,71 @@ +# reusable variables +fre_properties: + - &AM5_VERSION "am5f7b12r1" + - &FRE_STEM !join [am5/, *AM5_VERSION] + + # amip + - &EXP_AMIP_START "19790101T0000Z" + - &EXP_AMIP_END "20200101T0000Z" + - &ANA_AMIP_START "19800101T0000Z" + - &ANA_AMIP_END "20200101T0000Z" + + - &PP_AMIP_CHUNK96 "P1Y" + - &PP_AMIP_CHUNK384 "P1Y" + - &PP_XYINTERP96 "180,288" + - &PP_XYINTERP384 "720,1152" + + # climo + - &EXP_CLIMO_START96 "0001" + - &EXP_CLIMO_END96 "0011" + - &ANA_CLIMO_START96 "0002" + - &ANA_CLIMO_END96 "0011" + + - &EXP_CLIMO_START384 "0001" + - &EXP_CLIMO_END384 "0006" + - &ANA_CLIMO_START384 "0002" + - &ANA_CLIMO_END384 "0006" + + # coupled + - &PP_CPLD_CHUNK_A "P5Y" + - &PP_CPLD_CHUNK_B "P20Y" + + # grids + - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + - &release "f1a1r1" + - &INTEL "intel-classic" + - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +# compile information +build: + compileYaml: "compile.yaml" + platformYaml: "yaml_include/platforms.yaml" + +experiments: + - name: "c96L65_am5f7b12r1_amip_TESTING" + settings: "yaml_include/settings.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + - "yaml_include/pp-test.c96_amip.yaml" + - name: "c96L65_am5f7b12r1_amip_TESTING_WRONG" + settings: "yaml_include/settings_WRONG.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + - name: "c96L65_am5f7b12r1_pdclim1850F" + pp: + - "yaml_include/pp.c96_clim.yaml" + +# amip: +# settings: +# - shared/settings.yaml +# - shared/directories.yaml +# run: +# version: 1.1 +# - run/inputs.yaml +# - run/runtime.yaml +# postprocess: +# version: 2.0 +# - pp/components +# - analysis/legacy-bw.yaml diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml new file mode 100644 index 000000000..76f078523 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml @@ -0,0 +1,33 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "200,200" + +#c96_amip_postprocess: +postprocess: + components: + - type: "atmos_cmip-TEST" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos-TEST" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level_cmip-TEST" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..d20d05757 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,94 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "180,360" + +#c96_amip_postprocess: +postprocess: + # main pp instructions + components: + - type: "atmos_cmip" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + - type: "atmos_level_cmip" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_month_aer" + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_diurnal" + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_scalar" + sources: + - history_file: "atmos_scalar" + postprocess_on: True + - type: "aerosol_cmip" + xyInterp: *PP_XYINTERP96 + sources: + - history_file: "aerosol_month_cmip" + sourceGrid: "cubedsphere" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "land" + sources: + - history_file: "land_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "land_cmip" + sources: + - history_file: "land_month_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "tracer_level" + sources: + - history_file: "atmos_tracer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml new file mode 100644 index 000000000..791c0c437 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -0,0 +1,32 @@ +#workflow repositories +workflow: + run_workflow: + repo: "https://github.com/NOAA-GFDL" + version: "tbd" + pp_workflow: + repo: "https://github.com/NOAA-GFDL/fre-workflows.git" + version: "main" + +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml new file mode 100644 index 000000000..c973591e8 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -0,0 +1,32 @@ +#workflow repositories +workflow: + run_workflow: + repo: "https://github.com/NOAA-GFDL" + version: "tbd" + pp_workflow: + repo: + version: "main" + +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py new file mode 100644 index 000000000..997e7c5ab --- /dev/null +++ b/fre/workflow/tests/test_checkout_script.py @@ -0,0 +1,147 @@ +''' fre workflow checkout tests ''' +import stat +import re +import os +from pathlib import Path +import pytest +from fre.workflow import checkout_script + +TEST_CONFIGS = "fre/workflow/tests/AM5_example/" +EXPERIMENT = "c96L65_am5f7b12r1_amip_TESTING" + +@pytest.fixture(autouse=True, name="fake_home") +def fake_home_fixture(tmp_path, monkeypatch): + """ + Set the tmp_path as HOME for the cylc-src directory + to be created in. + """ + ## Mock HOME for cylc-src and cylc-run + fake_home = Path(tmp_path) + monkeypatch.setenv("HOME", str(fake_home)) + + return fake_home + +def test_cylc_src_creation_fail(fake_home): + """ + Test for the expected failure if the cylc-src + directory cannot be created. + + This test simulates a permission error in HOME. + """ + # Temporarily change fake_home permissions to read-only, so cylc-src creation will fail + os.chmod(fake_home, stat.S_IREAD) + + # run checkout to create cylc-src + directory = f"{fake_home}/cylc-src" + expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" + with pytest.raises(OSError, match = re.escape(expected_error)): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + +def test_check_missing_repo(): + """ + Test for the expected ValueError if the repo is not + defined in the settings.yaml. + """ + experiment = "c96L65_am5f7b12r1_amip_TESTING_WRONG" + repo = None + tag = "main" + with pytest.raises(ValueError, match = f"One of these are None: repo / tag = {repo} / {tag}"): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = experiment, + application = "pp", + branch = None) + +#def test_run_workflow_checkout(caplog): +# """ +# Test for a successful run workflow checkout. +# """ +# checkout_script.workflow_checkout(yamlfile, +# experiment = "c96L65_am5f7b12r1_amip_TESTING", +# application = "run" +# branch = None) + +def test_pp_workflow_checkout(fake_home, caplog): + """ + Test for a successful post-processing workflow checkout. + """ + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + + expected_repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + expected_tag = "main" + assert all([Path(f"{fake_home}/cylc-src/{EXPERIMENT}").exists(), + Path(f"{fake_home}/cylc-src/{EXPERIMENT}").is_dir(), + Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), + Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").is_file(), + f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) + +def test_pp_workflow_checkout_exists_already(fake_home, caplog): + """ + Test for the expected output message if the checkout/branch already exists. + """ + # 1st checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + + # 2nd checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + + repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + expected_tag = "main" + expected_output = [ + f"({repo}):({expected_tag}) check out ==> REQUESTED", + (f"Checkout exists ('{fake_home}/cylc-src/c96L65_am5f7b12r1_amip_TESTING'), " + f"and matches '{expected_tag}'") + ] + + for string in expected_output: + assert string in caplog.text + +def test_pp_workflow_checkout_branch_override(caplog): + """ + Test for correct checkout if a '-b', '--branch' is specified. + """ + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = "2025.04") + + repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + expected_tag = "2025.04" + expected_output = [f"({repo}):({expected_tag}) check out ==> REQUESTED", + f"({repo}):({expected_tag}) check out ==> SUCCESSFUL"] + + for string in expected_output: + assert string in caplog.text + +def test_pp_workflow_checkout_branch_conflict(fake_home, caplog): + """ + Test for the expected ValueError if the checkout was done already, + but user is checking out same repo again, with a different branch/tag. + """ + # 1st checkout: using default 'main' branch as set in the settings.yaml (version) + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = None) + + expected_output = (f"ERROR: Checkout exists ('{fake_home}/cylc-src/{EXPERIMENT}') " + "and does not match '2025.04'") + expected_error = "Neither tag nor branch matches the git clone branch arg" + with pytest.raises(ValueError, match = expected_error): + # 2nd checkout: trying to check out 2025.04 tag while 'main' already checked out + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + branch = "2025.04") + assert expected_output in caplog.text From c7cbbfed74a9d2d24bee794c472883049a3340a8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 27 Jan 2026 15:44:32 -0500 Subject: [PATCH 062/119] #697 Comment out non-existent tools for now --- fre/workflow/freworkflow.py | 98 ++++++++++++++----------------------- 1 file changed, 38 insertions(+), 60 deletions(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 21b6fec44..3fb4d57ba 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -20,10 +20,6 @@ def workflow_cli(): @click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -#@click.option("-p", "--platform", type=str, -# help="Platform name") -#@click.option("-t", "--target", type=str, -# help="Target name") @click.option("-b", "--branch", type =str, required=False, default = None, help="fre-workflows branch/tag to clone; default is $(fre --version)") @@ -37,65 +33,47 @@ def checkout(yamlfile, experiment, application, branch=None): """ checkout_script.workflow_checkout(yamlfile, experiment, application, branch) -@workflow_cli.command() -@click.option("-e", "--experiment", type=str, - help="Experiment name", - required=True) -#@click.option("-p", "--platform", type=str, -# help="Platform name", -# required=True) -#@click.option("-t", "--target", type=str, -# help="Target name", +#@workflow_cli.command() +#@click.option("-e", "--experiment", type=str, +# help="Experiment name", # required=True) -def install(experiment): - """ - Install workflow configuration - """ - install_script.workflow_install(experiment) +#def install(experiment): +# """ +# Install workflow configuration +# """ +# install_script.workflow_install(experiment) -@workflow_cli.command() -@click.option("-e", "--experiment", type=str, - help="Experiment name", - required=True) -#@click.option("-p", "--platform", type=str, -# help="Platform name", +#@workflow_cli.command() +#@click.option("-e", "--experiment", type=str, +# help="Experiment name", # required=True) -#@click.option("-t", "--target", type=str, -# help="Target name", -# required=True) -@click.option("--pause", is_flag=True, default=False, - help="Pause the workflow immediately on start up", - required=False) -@click.option("--no_wait", is_flag=True, default=False, - help="after submission, do not wait to ping the scheduler and confirm success", - required=False) -def run(experiment, pause, no_wait): - """ - Run workflow configuration - """ - run_script.workflow_run(experiment, pause, no_wait) +#@click.option("--pause", is_flag=True, default=False, +# help="Pause the workflow immediately on start up", +# required=False) +#@click.option("--no_wait", is_flag=True, default=False, +# help="after submission, do not wait to ping the scheduler and confirm success", +# required=False) +#def run(experiment, pause, no_wait): +# """ +# Run workflow configuration +# """ +# run_script.workflow_run(experiment, pause, no_wait) -@workflow_cli.command() -@click.option("-e", "--experiment", type=str, - help="Experiment name", - required=True) -#@click.option("-p", "--platform", type=str, -# help="Platform name", +#@workflow_cli.command() +#@click.option("-e", "--experiment", type=str, +# help="Experiment name", # required=True) -#@click.option("-T", "--target", type=str, -# help="Target name", +#@click.option("-c", "--config-file", type=str, +# help="Path to a configuration file in either XML or YAML", # required=True) -@click.option("-c", "--config-file", type=str, - help="Path to a configuration file in either XML or YAML", - required=True) -@click.option("-b", "--branch", - required=False, default=None, - help="fre-workflows branch/tag to clone; default is $(fre --version)") -@click.option("-t", "--time", - required=False, default=None, - help="Time whose history files are ready") -def all(experiment, platform, target, config_file, branch, time): - """ - Execute all fre workflow initialization steps in order - """ - wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) +#@click.option("-b", "--branch", +# required=False, default=None, +# help="fre-workflows branch/tag to clone; default is $(fre --version)") +#@click.option("-t", "--time", +# required=False, default=None, +# help="Time whose history files are ready") +#def all(experiment, platform, target, config_file, branch, time): +# """ +# Execute all fre workflow initialization steps in order +# """ +# wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) From db193b9ceefb06156b161dcd6f1ae82fd4e925a4 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 6 Feb 2026 14:21:50 -0500 Subject: [PATCH 063/119] #697 Comment out things that aren't done yet --- fre/workflow/freworkflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 3fb4d57ba..87bdee045 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -6,8 +6,8 @@ #fre tools from . import checkout_script -from . import install_script -from . import run_script +#from . import install_script +#from . import run_script @click.group(help=click.style(" - workflow subcommands", fg=(57,139,210))) def workflow_cli(): From 96a1f7ab25dd7adcda019bcffc1daa3e5583b47c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 6 Feb 2026 14:22:16 -0500 Subject: [PATCH 064/119] #697 Add validation and update output file name for resolved yaml --- fre/workflow/checkout_script.py | 53 ++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 58539b1fd..cec0ebf85 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -3,16 +3,59 @@ import subprocess from pathlib import Path import logging +import shutil import fre.yamltools.combine_yamls_script as cy from fre.app.helpers import change_directory #from . import make_workflow_name +from jsonschema import validate, SchemaError, ValidationError fre_logger = logging.getLogger(__name__) FRE_PP_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" FRE_RUN_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" +######VALIDATE##### +def validate_yaml(yamlfile: dict, application: str) -> None: + """ + Validate the format of the yaml file based + on the schema.json in gfdl_msd_schemas + + :param yamlfile: Model, settings, pp, and analysis yaml + information combined into a dictionary + :type yamlfile: dict + :param application: ------------------------------------------------ + :type application: string + :raises ValueError: + - if gfdl_mdf_schema path is not valid + - combined yaml is not valid + - unclear error in validation + :return: None + :rtype: None + """ + schema_dir = Path(__file__).resolve().parents[1] + schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json') + fre_logger.info("Using yaml schema '%s'", schema_path) + # Load the json schema: .load() (vs .loads()) reads and parses the json in one) + try: + with open(schema_path,'r', encoding='utf-8') as s: + schema = json.load(s) + except: + fre_logger.error("Schema '%s' is not valid. Contact the FRE team.", schema_path) + raise + + # Validate yaml + # If the yaml is not valid, the schema validation will raise errors and exit + try: + validate(instance = yamlfile,schema=schema) + fre_logger.info("Combined yaml valid") + except SchemaError as exc: + raise ValueError(f"Schema '{schema_path}' is not valid. Contact the FRE team.") from exc + except ValidationError as exc: + raise ValueError("Combined yaml is not valid. Please fix the errors and try again.") from exc + except Exception as exc: + raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc + def workflow_checkout(yamlfile: str = None, experiment = None, application = None, branch = None): """ @@ -51,6 +94,7 @@ def workflow_checkout(yamlfile: str = None, experiment = None, target=target, use="run", output=None) + #validate_yaml(yamlfile = yaml, application = "run") workflow_info = yaml.get("workflow").get("run_workflow") elif application == "pp": # will probably be taken out and put above is "use" @@ -60,7 +104,8 @@ def workflow_checkout(yamlfile: str = None, experiment = None, platform=platform, target=target, use="pp", - output=None) + output=f"config.yaml") + #validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp_workflow") repo = workflow_info.get("repo") @@ -118,3 +163,9 @@ def workflow_checkout(yamlfile: str = None, experiment = None, fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') + + ## Move combined yaml to cylc-src location + cylc_src_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{experiment}") +# outfile = os.path.join(cylc_src_dir, f"{experiment}.yaml") + shutil.move(f"config.yaml", cylc_src_dir) + fre_logger.info("Combined yaml file moved to ~/cylc-src/%s", experiment) From 3bca7db717fdda04df7611783fc079bb057f1720 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 6 Feb 2026 14:41:58 -0500 Subject: [PATCH 065/119] #697 Add assert to test --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 997e7c5ab..7ef182aaa 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -77,7 +77,7 @@ def test_pp_workflow_checkout(fake_home, caplog): assert all([Path(f"{fake_home}/cylc-src/{EXPERIMENT}").exists(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}").is_dir(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").is_file(), + Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists() f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) def test_pp_workflow_checkout_exists_already(fake_home, caplog): From 87b326f0a366ffcfc614365949cd59356452a5ae Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 14:23:01 -0500 Subject: [PATCH 066/119] #697 Add comma --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 7ef182aaa..138492b45 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -77,7 +77,7 @@ def test_pp_workflow_checkout(fake_home, caplog): assert all([Path(f"{fake_home}/cylc-src/{EXPERIMENT}").exists(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}").is_dir(), Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists() + Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists(), f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) def test_pp_workflow_checkout_exists_already(fake_home, caplog): From 1293aa8e260172408293e6a356209595047e753b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 14:52:59 -0500 Subject: [PATCH 067/119] #697 Put file moving in checkout creation --- fre/workflow/checkout_script.py | 12 ++++++------ fre/workflow/tests/test_checkout_script.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index cec0ebf85..68bd45c4d 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -143,6 +143,12 @@ def workflow_checkout(yamlfile: str = None, experiment = None, capture_output = True, text = True, check = True) fre_logger.debug(clone_output) fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + + ## Move combined yaml to cylc-src location + cylc_src_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{experiment}") + current_dir = Path.cwd() + shutil.move(Path(f"{current_dir}/config.yaml"), cylc_src_dir) + fre_logger.info("Combined yaml file moved to ~/cylc-src/%s", experiment) else: # the repo checkout does exist, scenarios 3 and 4. with change_directory(f"{directory}/{workflow_name}"): @@ -163,9 +169,3 @@ def workflow_checkout(yamlfile: str = None, experiment = None, fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') - - ## Move combined yaml to cylc-src location - cylc_src_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{experiment}") -# outfile = os.path.join(cylc_src_dir, f"{experiment}.yaml") - shutil.move(f"config.yaml", cylc_src_dir) - fre_logger.info("Combined yaml file moved to ~/cylc-src/%s", experiment) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 138492b45..3039ec21b 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -32,7 +32,7 @@ def test_cylc_src_creation_fail(fake_home): os.chmod(fake_home, stat.S_IREAD) # run checkout to create cylc-src - directory = f"{fake_home}/cylc-src" + directory = Path(f"{fake_home}/cylc-src") expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" with pytest.raises(OSError, match = re.escape(expected_error)): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", From 4a36008fb137ad7b24c03473b1e48c1e1baee141 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 15:20:54 -0500 Subject: [PATCH 068/119] #697 See if pipeline run as root --- fre/workflow/tests/test_checkout_script.py | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 3039ec21b..884f0c6cc 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -31,14 +31,21 @@ def test_cylc_src_creation_fail(fake_home): # Temporarily change fake_home permissions to read-only, so cylc-src creation will fail os.chmod(fake_home, stat.S_IREAD) - # run checkout to create cylc-src - directory = Path(f"{fake_home}/cylc-src") - expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" - with pytest.raises(OSError, match = re.escape(expected_error)): - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - branch = None) + try: + if os.geteuid == 0: # if running as root + pytest.skip("Cannot test premission errors when running as root") + print("RUNNING AS ROOT") + +# # run checkout to create cylc-src +# directory = Path(f"{fake_home}/cylc-src") +# expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" +# with pytest.raises(OSError, match = re.escape(expected_error)): +# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", +# experiment = EXPERIMENT, +# application = "pp", +# branch = None) + finally: + os.chmod(fake_home, stat.IRWXU) def test_check_missing_repo(): """ From 866b9d4ec21c7f1f4ff09712e3acec718a2bc914 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 16:23:18 -0500 Subject: [PATCH 069/119] #697 Fix stat --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 884f0c6cc..47a34bd0c 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -45,7 +45,7 @@ def test_cylc_src_creation_fail(fake_home): # application = "pp", # branch = None) finally: - os.chmod(fake_home, stat.IRWXU) + os.chmod(fake_home, stat.S_IRWXU) def test_check_missing_repo(): """ From c374c690baebf8bbe2fdf36ae19c082018fcf157 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 17:52:50 -0500 Subject: [PATCH 070/119] #697 make it fail for a sec --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 47a34bd0c..28c3d1018 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -35,7 +35,7 @@ def test_cylc_src_creation_fail(fake_home): if os.geteuid == 0: # if running as root pytest.skip("Cannot test premission errors when running as root") print("RUNNING AS ROOT") - + AH # # run checkout to create cylc-src # directory = Path(f"{fake_home}/cylc-src") # expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" From 57452a34b9836b3067c1622ec21b0e3dd91053d8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:23:48 -0500 Subject: [PATCH 071/119] test failure --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 28c3d1018..c8a654321 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -35,7 +35,7 @@ def test_cylc_src_creation_fail(fake_home): if os.geteuid == 0: # if running as root pytest.skip("Cannot test premission errors when running as root") print("RUNNING AS ROOT") - AH + AH # # run checkout to create cylc-src # directory = Path(f"{fake_home}/cylc-src") # expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" From 341c7d83b9b58f8c1c23095f04a12c0c91b4152a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:35:07 -0500 Subject: [PATCH 072/119] #697 Add some readme content --- fre/workflow/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index e69de29bb..51e748026 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -0,0 +1,18 @@ +# FRE workflow +`fre workflow` provides subtools that help to clone, install, and run a workflow from a repository. + +## Quickstart +From the root of the fre-cli repository, run: +``` +# List post-processing experiments defined in model yaml +fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING -a pp +``` + +## Subtools +- `fre workflow checkout [options]` + - Purpose: Clone the workflow repository/branch, depending on the application passed. + - Options: + - `-y, --yamlfile [model yaml] (str; required)` + - `-e, --experiment [experiment name] (str; required)` + - `-b, --branch [workflow repo branch to check out] (str; optional)` + - `-a, --application [ run | pp ] (str; required)` From 6928357851eadc6ceabc2d9d380b1101d634a10b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:36:47 -0500 Subject: [PATCH 073/119] #697 Update comment --- fre/workflow/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 51e748026..121d714c4 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -4,7 +4,7 @@ ## Quickstart From the root of the fre-cli repository, run: ``` -# List post-processing experiments defined in model yaml +# Checkout/clone the post-processing workflow repository fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING -a pp ``` From f4b7a7d0ff1f61f2e70c681132130f33423c7b93 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:42:36 -0500 Subject: [PATCH 074/119] #697 Remove click branch overwrite --- fre/workflow/README.md | 1 - fre/workflow/checkout_script.py | 19 +++---------------- fre/workflow/freworkflow.py | 7 ++----- 3 files changed, 5 insertions(+), 22 deletions(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 121d714c4..1006fc61e 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -14,5 +14,4 @@ fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7 - Options: - `-y, --yamlfile [model yaml] (str; required)` - `-e, --experiment [experiment name] (str; required)` - - `-b, --branch [workflow repo branch to check out] (str; optional)` - `-a, --application [ run | pp ] (str; required)` diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 68bd45c4d..222183ae0 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -56,8 +56,7 @@ def validate_yaml(yamlfile: dict, application: str) -> None: except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def workflow_checkout(yamlfile: str = None, experiment = None, - application = None, branch = None): +def workflow_checkout(yamlfile: str = None, experiment = None, application = None): """ Create a directory and clone the workflow template files from a defined repo. @@ -67,19 +66,11 @@ def workflow_checkout(yamlfile: str = None, experiment = None, yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None :type experiment: str - :param platform: The location + compiler that was used to run the model - (e.g. gfdl.ncrc5-deploy), default None - :type platform: str - :param target: Options used for the model compiler (e.g. prod-openmp), default None - :type target: str - :param branch: which git branch to pull from, default None - :type branch: str :param application: Which workflow will be used/cloned :type application: str :raises OSError: why checkout script was not able to be created :raises ValueError: -if experiment or platform or target is None - -if branch argument cannot be found as a branch or tag """ # Used in consolidate_yamls function for now platform = None @@ -110,12 +101,8 @@ def workflow_checkout(yamlfile: str = None, experiment = None, repo = workflow_info.get("repo") - if not branch: - tag = workflow_info.get("version") - fre_logger.info("Default tag ==> '%s'", tag) - else: - tag = branch - fre_logger.info("Requested branch/tag ==> '%s'", tag) + tag = workflow_info.get("version") + fre_logger.info("Defined tag ==> '%s'", tag) if None in [repo, tag]: raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}") diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 87bdee045..ac0964cf4 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -20,18 +20,15 @@ def workflow_cli(): @click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -@click.option("-b", "--branch", type =str, - required=False, default = None, - help="fre-workflows branch/tag to clone; default is $(fre --version)") @click.option("-a", "--application", type=click.Choice(['run', 'pp']), help="Use case for checked out workflow", required=True) -def checkout(yamlfile, experiment, application, branch=None): +def checkout(yamlfile, experiment, application): """ Checkout/extract fre workflow """ - checkout_script.workflow_checkout(yamlfile, experiment, application, branch) + checkout_script.workflow_checkout(yamlfile, experiment, application) #@workflow_cli.command() #@click.option("-e", "--experiment", type=str, From 6725ce8c626fc340faa00f71dadf865d19a3defc Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 09:49:33 -0500 Subject: [PATCH 075/119] #697 Comment out use of branch override --- fre/workflow/tests/test_checkout_script.py | 94 ++++++++++------------ 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index c8a654321..7f4eb711b 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -42,8 +42,7 @@ def test_cylc_src_creation_fail(fake_home): # with pytest.raises(OSError, match = re.escape(expected_error)): # checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", # experiment = EXPERIMENT, -# application = "pp", -# branch = None) +# application = "pp") finally: os.chmod(fake_home, stat.S_IRWXU) @@ -58,8 +57,7 @@ def test_check_missing_repo(): with pytest.raises(ValueError, match = f"One of these are None: repo / tag = {repo} / {tag}"): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = experiment, - application = "pp", - branch = None) + application = "pp") #def test_run_workflow_checkout(caplog): # """ @@ -67,8 +65,7 @@ def test_check_missing_repo(): # """ # checkout_script.workflow_checkout(yamlfile, # experiment = "c96L65_am5f7b12r1_amip_TESTING", -# application = "run" -# branch = None) +# application = "run") def test_pp_workflow_checkout(fake_home, caplog): """ @@ -76,8 +73,7 @@ def test_pp_workflow_checkout(fake_home, caplog): """ checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp", - branch = None) + application = "pp") expected_repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" @@ -94,14 +90,12 @@ def test_pp_workflow_checkout_exists_already(fake_home, caplog): # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp", - branch = None) + application = "pp") # 2nd checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp", - branch = None) + application = "pp") repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" @@ -114,41 +108,41 @@ def test_pp_workflow_checkout_exists_already(fake_home, caplog): for string in expected_output: assert string in caplog.text -def test_pp_workflow_checkout_branch_override(caplog): - """ - Test for correct checkout if a '-b', '--branch' is specified. - """ - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - branch = "2025.04") - - repo = "https://github.com/NOAA-GFDL/fre-workflows.git" - expected_tag = "2025.04" - expected_output = [f"({repo}):({expected_tag}) check out ==> REQUESTED", - f"({repo}):({expected_tag}) check out ==> SUCCESSFUL"] - - for string in expected_output: - assert string in caplog.text - -def test_pp_workflow_checkout_branch_conflict(fake_home, caplog): - """ - Test for the expected ValueError if the checkout was done already, - but user is checking out same repo again, with a different branch/tag. - """ - # 1st checkout: using default 'main' branch as set in the settings.yaml (version) - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - branch = None) - - expected_output = (f"ERROR: Checkout exists ('{fake_home}/cylc-src/{EXPERIMENT}') " - "and does not match '2025.04'") - expected_error = "Neither tag nor branch matches the git clone branch arg" - with pytest.raises(ValueError, match = expected_error): - # 2nd checkout: trying to check out 2025.04 tag while 'main' already checked out - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - branch = "2025.04") - assert expected_output in caplog.text +#def test_pp_workflow_checkout_branch_override(caplog): +# """ +# Test for correct checkout if a '-b', '--branch' is specified. +# """ +# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", +# experiment = EXPERIMENT, +# application = "pp", +# branch = "2025.04") +# +# repo = "https://github.com/NOAA-GFDL/fre-workflows.git" +# expected_tag = "2025.04" +# expected_output = [f"({repo}):({expected_tag}) check out ==> REQUESTED", +# f"({repo}):({expected_tag}) check out ==> SUCCESSFUL"] +# +# for string in expected_output: +# assert string in caplog.text +# +#def test_pp_workflow_checkout_branch_conflict(fake_home, caplog): +# """ +# Test for the expected ValueError if the checkout was done already, +# but user is checking out same repo again, with a different branch/tag. +# """ +# # 1st checkout: using default 'main' branch as set in the settings.yaml (version) +# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", +# experiment = EXPERIMENT, +# application = "pp", +# branch = None) +# +# expected_output = (f"ERROR: Checkout exists ('{fake_home}/cylc-src/{EXPERIMENT}') " +# "and does not match '2025.04'") +# expected_error = "Neither tag nor branch matches the git clone branch arg" +# with pytest.raises(ValueError, match = expected_error): +# # 2nd checkout: trying to check out 2025.04 tag while 'main' already checked out +# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", +# experiment = EXPERIMENT, +# application = "pp", +# branch = "2025.04") +# assert expected_output in caplog.text From 169f0483cd67c80944a22a22aa0b97b298b3f836 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 11:26:53 -0500 Subject: [PATCH 076/119] #697 Fix file permission test --- fre/workflow/tests/test_checkout_script.py | 25 ++++++++++------------ 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 7f4eb711b..3f19ffa8e 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -28,21 +28,18 @@ def test_cylc_src_creation_fail(fake_home): This test simulates a permission error in HOME. """ - # Temporarily change fake_home permissions to read-only, so cylc-src creation will fail - os.chmod(fake_home, stat.S_IREAD) - try: - if os.geteuid == 0: # if running as root - pytest.skip("Cannot test premission errors when running as root") - print("RUNNING AS ROOT") - AH -# # run checkout to create cylc-src -# directory = Path(f"{fake_home}/cylc-src") -# expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" -# with pytest.raises(OSError, match = re.escape(expected_error)): -# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", -# experiment = EXPERIMENT, -# application = "pp") + # Temporarily change fake_home permissions to read-only for usr/owner + # If read-only, cylc-src dir creation should fail + Path(fake_home).chmod(stat.S_IRUSR) + + # run checkout to create cylc-src + directory = Path(f"{fake_home}/cylc-src") + expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" + with pytest.raises(OSError, match = re.escape(expected_error)): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp") finally: os.chmod(fake_home, stat.S_IRWXU) From 1f849934d3b4b7aa81ed37ca04cf12d5e27d0169 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 11:27:56 -0500 Subject: [PATCH 077/119] #697 Fix test --- fre/workflow/tests/test_checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 3f19ffa8e..22b0b158e 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -41,7 +41,7 @@ def test_cylc_src_creation_fail(fake_home): experiment = EXPERIMENT, application = "pp") finally: - os.chmod(fake_home, stat.S_IRWXU) + Path(fake_home).chmod(stat.S_IRWXU) def test_check_missing_repo(): """ From 56b4741a023b94e5f11ad703def58dbb149a2ecf Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 11:47:42 -0500 Subject: [PATCH 078/119] #697 hopfeully change file permissions correctly this time --- fre/workflow/tests/test_checkout_script.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 22b0b158e..bb609aa52 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -31,7 +31,8 @@ def test_cylc_src_creation_fail(fake_home): try: # Temporarily change fake_home permissions to read-only for usr/owner # If read-only, cylc-src dir creation should fail - Path(fake_home).chmod(stat.S_IRUSR) + read_only_mode = 0o44 + Path(fake_home).chmod(read_only_mode) # run checkout to create cylc-src directory = Path(f"{fake_home}/cylc-src") @@ -41,7 +42,8 @@ def test_cylc_src_creation_fail(fake_home): experiment = EXPERIMENT, application = "pp") finally: - Path(fake_home).chmod(stat.S_IRWXU) + rwx_mode = 0o77 + Path(fake_home).chmod(rwx_mode) def test_check_missing_repo(): """ From 8997960e31b24ca2a3bd2ea2fa74c0b6e5c4c124 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 15:40:02 -0500 Subject: [PATCH 079/119] #697 Nix file permission changes - doesn't work when running as root --- fre/workflow/tests/test_checkout_script.py | 25 +++++++++------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index bb609aa52..76921cc6e 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -28,22 +28,17 @@ def test_cylc_src_creation_fail(fake_home): This test simulates a permission error in HOME. """ - try: - # Temporarily change fake_home permissions to read-only for usr/owner - # If read-only, cylc-src dir creation should fail - read_only_mode = 0o44 - Path(fake_home).chmod(read_only_mode) + cylc_src_file = Path(f"{fake_home}/cylc-src") + with open(cylc_src_file, "w") as f: + f.write("testing 123") - # run checkout to create cylc-src - directory = Path(f"{fake_home}/cylc-src") - expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" - with pytest.raises(OSError, match = re.escape(expected_error)): - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp") - finally: - rwx_mode = 0o77 - Path(fake_home).chmod(rwx_mode) + # run checkout to create cylc-src + directory = Path(f"{fake_home}/cylc-src") + expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" + with pytest.raises(OSError, match = re.escape(expected_error)): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp") def test_check_missing_repo(): """ From 8a80800d2afa73e4b4b519bdb5cfc9edfb86211c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 15:56:11 -0500 Subject: [PATCH 080/119] #697 Address pylint messages --- fre/workflow/tests/test_checkout_script.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 76921cc6e..c4a0190b6 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -1,7 +1,5 @@ ''' fre workflow checkout tests ''' -import stat import re -import os from pathlib import Path import pytest from fre.workflow import checkout_script @@ -26,10 +24,11 @@ def test_cylc_src_creation_fail(fake_home): Test for the expected failure if the cylc-src directory cannot be created. - This test simulates a permission error in HOME. + This test simulates a file with the name cylc-src + already created, causing a permission error in HOME. """ cylc_src_file = Path(f"{fake_home}/cylc-src") - with open(cylc_src_file, "w") as f: + with open(cylc_src_file, "w", encoding='utf-8') as f: f.write("testing 123") # run checkout to create cylc-src From 637e6ee983b99b5a433b55b4131f71af62aea486 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Feb 2026 17:33:51 -0500 Subject: [PATCH 081/119] #697 Add fre workflow cli tests --- fre/tests/test_fre_workflow_cli.py | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 fre/tests/test_fre_workflow_cli.py diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py new file mode 100644 index 000000000..def167d2d --- /dev/null +++ b/fre/tests/test_fre_workflow_cli.py @@ -0,0 +1,36 @@ +import os +import shutil +from pathlib import Path + +from click.testing import CliRunner + +from fre import fre + +""" +CLI Tests for fre workflow * +Tests the command-line-interface calls for tools in the fre workflow suite. +Each tool generally gets 3 tests: + - fre workflow $tool, checking for exit code 0 (fails if cli isn't configured right) + - fre workflow $tool --help, checking for exit code 0 (fails if the code doesn't run) + - fre workflow $tool --optionDNE, checking for exit code 2 (fails if cli isn't configured + right and thinks the tool has a --optionDNE option) +""" + +runner = CliRunner() + +#-- fre pp +def test_cli_fre_workflow(): + ''' fre workflow ''' + result = runner.invoke(fre.fre, args=["workflow"]) + assert result.exit_code == 2 + +def test_cli_fre_workflow_help(): + ''' fre workflow --help ''' + result = runner.invoke(fre.fre, args=["workflow", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_workflow_opt_dne(): + ''' fre workflow optionDNE ''' + result = runner.invoke(fre.fre, args=["workflow", "optionDNE"]) + assert result.exit_code == 2 + From 318a426ea2de5510f2debce132314ea77324289b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 19 Feb 2026 16:48:11 -0500 Subject: [PATCH 082/119] #697 Some clean up --- fre/workflow/__init__.py | 27 ------------- fre/workflow/checkout_script.py | 1 - fre/workflow/freworkflow.py | 45 ---------------------- fre/workflow/tests/test_checkout_script.py | 39 ------------------- 4 files changed, 112 deletions(-) diff --git a/fre/workflow/__init__.py b/fre/workflow/__init__.py index dc62bb794..e69de29bb 100644 --- a/fre/workflow/__init__.py +++ b/fre/workflow/__init__.py @@ -1,27 +0,0 @@ -from typing import Optional - -def make_workflow_name(experiment : Optional[str] = None) -> str: - """ - Function that takes in a triplet of tags for a model experiment, platform, and target, and - returns a directory name for the corresponding pp workflow. Because this is often given by - user to the shell being used by python, we split/reform the string to remove semi-colons or - spaces that may be used to execute an arbitrary command with elevated privileges. - - :param experiment: One of the postprocessing experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None - :type experiment: str - :param platform: The location + compiler that was used to run the model (e.g. gfdl.ncrc5-deploy), default None - :type platform: str - :param target: Options used for the model compiler (e.g. prod-openmp), default None - :type target: str - :return: string created in specific format from the input strings - :rtype: str - - .. note:: if any arguments are None, then "None" will appear in the workflow name - """ - name = f'{experiment}__{platform}__{target}' - return ''.join( - (''.join( - name.split(' ') - ) - ).split(';') - ) # user-input sanitation, prevents some malicious cmds from being executed with privileges diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 222183ae0..17351aff7 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -7,7 +7,6 @@ import fre.yamltools.combine_yamls_script as cy from fre.app.helpers import change_directory -#from . import make_workflow_name from jsonschema import validate, SchemaError, ValidationError fre_logger = logging.getLogger(__name__) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index ac0964cf4..55d1d15bf 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -29,48 +29,3 @@ def checkout(yamlfile, experiment, application): Checkout/extract fre workflow """ checkout_script.workflow_checkout(yamlfile, experiment, application) - -#@workflow_cli.command() -#@click.option("-e", "--experiment", type=str, -# help="Experiment name", -# required=True) -#def install(experiment): -# """ -# Install workflow configuration -# """ -# install_script.workflow_install(experiment) - -#@workflow_cli.command() -#@click.option("-e", "--experiment", type=str, -# help="Experiment name", -# required=True) -#@click.option("--pause", is_flag=True, default=False, -# help="Pause the workflow immediately on start up", -# required=False) -#@click.option("--no_wait", is_flag=True, default=False, -# help="after submission, do not wait to ping the scheduler and confirm success", -# required=False) -#def run(experiment, pause, no_wait): -# """ -# Run workflow configuration -# """ -# run_script.workflow_run(experiment, pause, no_wait) - -#@workflow_cli.command() -#@click.option("-e", "--experiment", type=str, -# help="Experiment name", -# required=True) -#@click.option("-c", "--config-file", type=str, -# help="Path to a configuration file in either XML or YAML", -# required=True) -#@click.option("-b", "--branch", -# required=False, default=None, -# help="fre-workflows branch/tag to clone; default is $(fre --version)") -#@click.option("-t", "--time", -# required=False, default=None, -# help="Time whose history files are ready") -#def all(experiment, platform, target, config_file, branch, time): -# """ -# Execute all fre workflow initialization steps in order -# """ -# wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index c4a0190b6..5cfc6e189 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -100,42 +100,3 @@ def test_pp_workflow_checkout_exists_already(fake_home, caplog): for string in expected_output: assert string in caplog.text - -#def test_pp_workflow_checkout_branch_override(caplog): -# """ -# Test for correct checkout if a '-b', '--branch' is specified. -# """ -# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", -# experiment = EXPERIMENT, -# application = "pp", -# branch = "2025.04") -# -# repo = "https://github.com/NOAA-GFDL/fre-workflows.git" -# expected_tag = "2025.04" -# expected_output = [f"({repo}):({expected_tag}) check out ==> REQUESTED", -# f"({repo}):({expected_tag}) check out ==> SUCCESSFUL"] -# -# for string in expected_output: -# assert string in caplog.text -# -#def test_pp_workflow_checkout_branch_conflict(fake_home, caplog): -# """ -# Test for the expected ValueError if the checkout was done already, -# but user is checking out same repo again, with a different branch/tag. -# """ -# # 1st checkout: using default 'main' branch as set in the settings.yaml (version) -# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", -# experiment = EXPERIMENT, -# application = "pp", -# branch = None) -# -# expected_output = (f"ERROR: Checkout exists ('{fake_home}/cylc-src/{EXPERIMENT}') " -# "and does not match '2025.04'") -# expected_error = "Neither tag nor branch matches the git clone branch arg" -# with pytest.raises(ValueError, match = expected_error): -# # 2nd checkout: trying to check out 2025.04 tag while 'main' already checked out -# checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", -# experiment = EXPERIMENT, -# application = "pp", -# branch = "2025.04") -# assert expected_output in caplog.text From a1e96c1d513f75e56819ca13236c10f6f9ce9907 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 14:37:52 -0500 Subject: [PATCH 083/119] #697 Add --force-checkout and --target-dir --- fre/workflow/checkout_script.py | 91 ++++++++++++++++++++++----------- fre/workflow/freworkflow.py | 14 +++-- 2 files changed, 72 insertions(+), 33 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 17351aff7..d9b05ca61 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -3,17 +3,16 @@ import subprocess from pathlib import Path import logging -import shutil +import shutil +import json +from jsonschema import validate, SchemaError, ValidationError +from typing import Optional import fre.yamltools.combine_yamls_script as cy from fre.app.helpers import change_directory -from jsonschema import validate, SchemaError, ValidationError fre_logger = logging.getLogger(__name__) -FRE_PP_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" -FRE_RUN_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git" - ######VALIDATE##### def validate_yaml(yamlfile: dict, application: str) -> None: """ @@ -55,7 +54,28 @@ def validate_yaml(yamlfile: dict, application: str) -> None: except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def workflow_checkout(yamlfile: str = None, experiment = None, application = None): +def create_checkout(repo, tag, src_dir, workflow_name): + """ + Create a directory and clone the workflow template files from a defined repo. --------- + ........... + ........... + ........... + ........... + """ + # scenarios 1+2, checkout doesn't exist, branch specified (or not) + clone_output = subprocess.run( ["git", "clone","--recursive", + f"--branch={tag}", + repo, f"{src_dir}/{workflow_name}"], + capture_output = True, text = True, check = True) + fre_logger.debug(clone_output) + fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + + ## Move combined yaml to cylc-src location + current_dir = Path.cwd() + shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") + fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) + +def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a defined repo. @@ -67,13 +87,18 @@ def workflow_checkout(yamlfile: str = None, experiment = None, application = Non :type experiment: str :param application: Which workflow will be used/cloned :type application: str - :raises OSError: why checkout script was not able to be created + :raises OSError: if the checkout script was not able to be created :raises ValueError: - -if experiment or platform or target is None + - if the repo and/or tag was not defined + - if the + raise ValueError('Neither tag nor branch matches the git clone branch arg') + """ # Used in consolidate_yamls function for now platform = None target = None +# print(src_dir) +# quit() if application == "run": fre_logger.info("NOT DONE YET") # will probably be taken out and put above is "use" @@ -94,7 +119,7 @@ def workflow_checkout(yamlfile: str = None, experiment = None, application = Non platform=platform, target=target, use="pp", - output=f"config.yaml") + output="config.yaml") #validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp_workflow") @@ -108,36 +133,42 @@ def workflow_checkout(yamlfile: str = None, experiment = None, application = Non fre_logger.warning("(%s):(%s) check out ==> REQUESTED", repo, tag) + # Make sure src_dir exists + if not Path(target_dir).exists(): + raise ValueError(f"Source directory {target_dir} does not exist or cannot be found.") + # clone directory - directory = os.path.expanduser("~/cylc-src") + src_dir = f"{target_dir}/cylc-src" # workflow name workflow_name = experiment # create workflow in cylc-src try: - Path(directory).mkdir(parents=True, exist_ok=True) + Path(src_dir).mkdir(parents=True, exist_ok=True) except Exception as exc: raise OSError( - f"(checkoutScript) directory {directory} wasn't able to be created. exit!") from exc + f"(checkoutScript) directory {src_dir} wasn't able to be created. exit!") from exc - if not Path(f"{directory}/{workflow_name}").is_dir(): - # scenarios 1+2, checkout doesn't exist, branch specified (or not) + if not Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.info("Workflow does not yet exist; will create now") - clone_output = subprocess.run( ["git", "clone","--recursive", - f"--branch={tag}", - repo, f"{directory}/{workflow_name}"], - capture_output = True, text = True, check = True) - fre_logger.debug(clone_output) - fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) - - ## Move combined yaml to cylc-src location - cylc_src_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{experiment}") - current_dir = Path.cwd() - shutil.move(Path(f"{current_dir}/config.yaml"), cylc_src_dir) - fre_logger.info("Combined yaml file moved to ~/cylc-src/%s", experiment) - else: + create_checkout(repo = repo, + tag = tag, + src_dir = src_dir, + workflow_name = workflow_name) + elif Path(f"{src_dir}/{workflow_name}").is_dir() and force_checkout: + fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) + # Remove checked out repo + shutil.rmtree(f"{src_dir}/{workflow_name}") + fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) + # Redo checkout + create_checkout(repo = repo, + tag = tag, + src_dir = src_dir, + workflow_name = workflow_name) + elif Path(f"{src_dir}/{workflow_name}").is_dir() and not force_checkout: + fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) # the repo checkout does exist, scenarios 3 and 4. - with change_directory(f"{directory}/{workflow_name}"): + with change_directory(f"{src_dir}/{workflow_name}"): # capture the branch and tag # if either match git_clone_branch_arg, then success. otherwise, fail. current_tag = subprocess.run(["git","describe","--tags"], @@ -148,10 +179,10 @@ def workflow_checkout(yamlfile: str = None, experiment = None, application = Non text = True, check = True).stdout.strip() if tag in (current_tag, current_branch): - fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", directory, workflow_name, tag) + fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) else: fre_logger.error( - "ERROR: Checkout exists ('%s/%s') and does not match '%s'", directory, workflow_name, tag) + "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag) fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 55d1d15bf..9360de710 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -1,5 +1,5 @@ ''' fre workflow ''' - +import os import click import logging fre_logger = logging.getLogger(__name__) @@ -24,8 +24,16 @@ def workflow_cli(): type=click.Choice(['run', 'pp']), help="Use case for checked out workflow", required=True) -def checkout(yamlfile, experiment, application): +@click.option("--target-dir", + type=str, + default=lambda: os.environ['TMPDIR'], + help=f"Target directory for workflow to be cloned into. Default location: {os.environ['TMPDIR']}") +@click.option("--force-checkout", + is_flag=True, + default=False, + help="If the checkout already, exists, remove and check out again.") +def checkout(yamlfile, experiment, application, target_dir, force_checkout): """ Checkout/extract fre workflow """ - checkout_script.workflow_checkout(yamlfile, experiment, application) + checkout_script.workflow_checkout(yamlfile, experiment, application, target_dir, force_checkout) From 322fb44150a0913aba56b294d28716267dd7962a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 14:38:28 -0500 Subject: [PATCH 084/119] #697 Update spacing to account for warning output --- fre/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/__init__.py b/fre/__init__.py index 3a3702c89..a7a2fa10c 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -9,7 +9,7 @@ fre_logger = logging.getLogger(__name__) -FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s" +FORMAT = "[%(levelname)7s:%(filename)24s:%(funcName)24s] %(message)s" logging.basicConfig(level = logging.WARNING, format = FORMAT, filename = None, From 78101e8bcef891d632c3c7426f1fa6c799c23285 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 15:20:53 -0500 Subject: [PATCH 085/119] #697 Update documentation --- fre/workflow/checkout_script.py | 33 +++++++++++++++++++++------------ fre/workflow/freworkflow.py | 2 +- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index d9b05ca61..ca16cc0ee 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -22,7 +22,7 @@ def validate_yaml(yamlfile: dict, application: str) -> None: :param yamlfile: Model, settings, pp, and analysis yaml information combined into a dictionary :type yamlfile: dict - :param application: ------------------------------------------------ + :param application: type of workflow to check out/clone :type application: string :raises ValueError: - if gfdl_mdf_schema path is not valid @@ -54,13 +54,19 @@ def validate_yaml(yamlfile: dict, application: str) -> None: except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def create_checkout(repo, tag, src_dir, workflow_name): +def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str) --> None: """ - Create a directory and clone the workflow template files from a defined repo. --------- - ........... - ........... - ........... - ........... + Clone the workflow template files from a defined repo into the cylc-src/workflow + directory and move the resolved yaml to the cylc-src directory. + + :param repo: Yaml defined workflow repository + :type repo: str + :param tag: branch or version of defined repository + :type tag: str + :param src_dir: Cylc-src directory + :type src_dir: str + :param workflow_name: Name of workflow + :type workflow_name: src """ # scenarios 1+2, checkout doesn't exist, branch specified (or not) clone_output = subprocess.run( ["git", "clone","--recursive", @@ -75,7 +81,7 @@ def create_checkout(repo, tag, src_dir, workflow_name): shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) -def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False): +def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False) --> None: """ Create a directory and clone the workflow template files from a defined repo. @@ -87,9 +93,14 @@ def workflow_checkout(yamlfile: str = None, experiment: str = None, application: :type experiment: str :param application: Which workflow will be used/cloned :type application: str + :param target_dir: Target directory to clone repository into + :type target_dir: str + :param force_checkout: re-clone the workflow repo if it exists + :type force_checkout: bool :raises OSError: if the checkout script was not able to be created :raises ValueError: - if the repo and/or tag was not defined + - if the target directory does not exist or cannot be found - if the raise ValueError('Neither tag nor branch matches the git clone branch arg') @@ -97,8 +108,7 @@ def workflow_checkout(yamlfile: str = None, experiment: str = None, application: # Used in consolidate_yamls function for now platform = None target = None -# print(src_dir) -# quit() + if application == "run": fre_logger.info("NOT DONE YET") # will probably be taken out and put above is "use" @@ -124,7 +134,6 @@ def workflow_checkout(yamlfile: str = None, experiment: str = None, application: workflow_info = yaml.get("workflow").get("pp_workflow") repo = workflow_info.get("repo") - tag = workflow_info.get("version") fre_logger.info("Defined tag ==> '%s'", tag) @@ -135,7 +144,7 @@ def workflow_checkout(yamlfile: str = None, experiment: str = None, application: # Make sure src_dir exists if not Path(target_dir).exists(): - raise ValueError(f"Source directory {target_dir} does not exist or cannot be found.") + raise ValueError(f"Target directory {target_dir} does not exist or cannot be found.") # clone directory src_dir = f"{target_dir}/cylc-src" diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 9360de710..06573c927 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -22,7 +22,7 @@ def workflow_cli(): required=True) @click.option("-a", "--application", type=click.Choice(['run', 'pp']), - help="Use case for checked out workflow", + help="Type of workflow to check out/clone", required=True) @click.option("--target-dir", type=str, From 560936d2ca439f7c531e82e62a62127805fd13c6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 15:32:13 -0500 Subject: [PATCH 086/119] #697 Update doc --- fre/workflow/checkout_script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index ca16cc0ee..e04a080fe 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -14,7 +14,7 @@ fre_logger = logging.getLogger(__name__) ######VALIDATE##### -def validate_yaml(yamlfile: dict, application: str) -> None: +def validate_yaml(yamlfile: dict, application: str): """ Validate the format of the yaml file based on the schema.json in gfdl_msd_schemas @@ -54,7 +54,7 @@ def validate_yaml(yamlfile: dict, application: str) -> None: except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str) --> None: +def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str): """ Clone the workflow template files from a defined repo into the cylc-src/workflow directory and move the resolved yaml to the cylc-src directory. @@ -81,7 +81,7 @@ def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str) --> N shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) -def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False) --> None: +def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a defined repo. From 9c23e7148c712cfde356a1f47dc6cd500dc8a7e8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 16:32:59 -0500 Subject: [PATCH 087/119] #697 Fix tests --- fre/workflow/checkout_script.py | 2 +- fre/workflow/tests/test_checkout_script.py | 77 ++++++++++++++-------- 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index e04a080fe..e368574a3 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -81,7 +81,7 @@ def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str): shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) -def workflow_checkout(yamlfile: str = None, experiment: str = None, application: str = None, target_dir: str = os.environ['TMPDIR'], force_checkout: Optional[bool] = False): +def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a defined repo. diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 5cfc6e189..a21c7b5ab 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -7,19 +7,19 @@ TEST_CONFIGS = "fre/workflow/tests/AM5_example/" EXPERIMENT = "c96L65_am5f7b12r1_amip_TESTING" -@pytest.fixture(autouse=True, name="fake_home") -def fake_home_fixture(tmp_path, monkeypatch): - """ - Set the tmp_path as HOME for the cylc-src directory - to be created in. - """ - ## Mock HOME for cylc-src and cylc-run - fake_home = Path(tmp_path) - monkeypatch.setenv("HOME", str(fake_home)) - - return fake_home - -def test_cylc_src_creation_fail(fake_home): +#@pytest.fixture(autouse=True, name="fake_home") +#def fake_home_fixture(tmp_path, monkeypatch): +# """ +# Set the tmp_path as HOME for the cylc-src directory +# to be created in. +# """ +# ## Mock HOME for cylc-src and cylc-run +# fake_home = Path(tmp_path) +# monkeypatch.setenv("HOME", str(fake_home)) +# +# return fake_home +# +def test_cylc_src_creation_fail(tmp_path): """ Test for the expected failure if the cylc-src directory cannot be created. @@ -27,19 +27,34 @@ def test_cylc_src_creation_fail(fake_home): This test simulates a file with the name cylc-src already created, causing a permission error in HOME. """ - cylc_src_file = Path(f"{fake_home}/cylc-src") + cylc_src_file = Path(f"{tmp_path}/cylc-src") with open(cylc_src_file, "w", encoding='utf-8') as f: f.write("testing 123") # run checkout to create cylc-src - directory = Path(f"{fake_home}/cylc-src") + directory = Path(f"{tmp_path}/cylc-src") expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" with pytest.raises(OSError, match = re.escape(expected_error)): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp") + application = "pp", + target_dir = tmp_path) -def test_check_missing_repo(): +def test_checkout_target_dir_dne(tmp_path): + """ + """ + bad_dir = "does/not/exist" + + # run checkout to create cylc-src + expected_error = f"Target directory {bad_dir} does not exist or cannot be found." + + with pytest.raises(ValueError, match = re.escape(expected_error)): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = bad_dir) + +def test_checkout_missing_repo(tmp_path): """ Test for the expected ValueError if the repo is not defined in the settings.yaml. @@ -50,7 +65,8 @@ def test_check_missing_repo(): with pytest.raises(ValueError, match = f"One of these are None: repo / tag = {repo} / {tag}"): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = experiment, - application = "pp") + application = "pp", + target_dir = tmp_path) #def test_run_workflow_checkout(caplog): # """ @@ -60,43 +76,48 @@ def test_check_missing_repo(): # experiment = "c96L65_am5f7b12r1_amip_TESTING", # application = "run") -def test_pp_workflow_checkout(fake_home, caplog): +def test_pp_workflow_checkout(tmp_path, caplog): """ Test for a successful post-processing workflow checkout. """ checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp") + application = "pp", + target_dir = tmp_path) expected_repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" - assert all([Path(f"{fake_home}/cylc-src/{EXPERIMENT}").exists(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}").is_dir(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), - Path(f"{fake_home}/cylc-src/{EXPERIMENT}/config.yaml").exists(), + assert all([Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").exists(), + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").is_dir(), + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/config.yaml").exists(), f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) -def test_pp_workflow_checkout_exists_already(fake_home, caplog): +def test_pp_workflow_checkout_exists_already(tmp_path, caplog): """ Test for the expected output message if the checkout/branch already exists. """ # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp") + application = "pp", + target_dir = tmp_path) # 2nd checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = EXPERIMENT, - application = "pp") + application = "pp", + target_dir = tmp_path) repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" expected_output = [ f"({repo}):({expected_tag}) check out ==> REQUESTED", - (f"Checkout exists ('{fake_home}/cylc-src/c96L65_am5f7b12r1_amip_TESTING'), " + (f"Checkout exists ('{tmp_path}/cylc-src/c96L65_am5f7b12r1_amip_TESTING'), " f"and matches '{expected_tag}'") ] for string in expected_output: assert string in caplog.text + +##def test_pp_workflow_checkout_force_checkout From 053c01ad82566991417ab09c238e7123ddf6b48c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 16:53:51 -0500 Subject: [PATCH 088/119] #697 Fix --target-dir click value - Use TMPDIR env variable if command line option not passed --- fre/workflow/freworkflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 06573c927..d45dc773d 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -26,8 +26,8 @@ def workflow_cli(): required=True) @click.option("--target-dir", type=str, - default=lambda: os.environ['TMPDIR'], - help=f"Target directory for workflow to be cloned into. Default location: {os.environ['TMPDIR']}") + envvar="TMPDIR", + help=f"Target directory for workflow to be cloned into. Default location TMPDIR: {os.environ['TMPDIR']}") @click.option("--force-checkout", is_flag=True, default=False, From a95469ee499aa9217df8f56438733f8636a7e520 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 17:11:39 -0500 Subject: [PATCH 089/119] #697 Add default dir (see if pipeline passes) --- fre/workflow/freworkflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index d45dc773d..61caf3d45 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -27,6 +27,7 @@ def workflow_cli(): @click.option("--target-dir", type=str, envvar="TMPDIR", + default=".fre" help=f"Target directory for workflow to be cloned into. Default location TMPDIR: {os.environ['TMPDIR']}") @click.option("--force-checkout", is_flag=True, From 6122a2cec00b352fdd9e13fd8e8c529ef5a46b77 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 17:42:30 -0500 Subject: [PATCH 090/119] #697 update --- fre/workflow/checkout_script.py | 4 ++-- fre/workflow/freworkflow.py | 2 +- fre/workflow/tests/AM5_example/yaml_include/settings.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index e368574a3..35af24d7e 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -120,7 +120,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N use="run", output=None) #validate_yaml(yamlfile = yaml, application = "run") - workflow_info = yaml.get("workflow").get("run_workflow") + workflow_info = yaml.get("workflow").get("run") elif application == "pp": # will probably be taken out and put above is "use" # is generalized in this tool @@ -131,7 +131,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N use="pp", output="config.yaml") #validate_yaml(yamlfile = yaml, application = "pp") - workflow_info = yaml.get("workflow").get("pp_workflow") + workflow_info = yaml.get("workflow").get("pp") repo = workflow_info.get("repo") tag = workflow_info.get("version") diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 61caf3d45..9b5f87bd1 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -27,7 +27,7 @@ def workflow_cli(): @click.option("--target-dir", type=str, envvar="TMPDIR", - default=".fre" + default=".fre", help=f"Target directory for workflow to be cloned into. Default location TMPDIR: {os.environ['TMPDIR']}") @click.option("--force-checkout", is_flag=True, diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml index 791c0c437..c45d9c533 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -1,9 +1,9 @@ #workflow repositories workflow: - run_workflow: + run: repo: "https://github.com/NOAA-GFDL" version: "tbd" - pp_workflow: + pp: repo: "https://github.com/NOAA-GFDL/fre-workflows.git" version: "main" From 8c5a8c2301678d905aa998da1e86b7c0a86142e4 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Feb 2026 17:57:03 -0500 Subject: [PATCH 091/119] #697 update --- .../tests/AM5_example/yaml_include/settings_WRONG.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml index c973591e8..312a5e7cd 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -1,9 +1,9 @@ #workflow repositories workflow: - run_workflow: + run: repo: "https://github.com/NOAA-GFDL" version: "tbd" - pp_workflow: + pp: repo: version: "main" From ea0d90f316f1a11dbdc921863f14ab1b8fa3419a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 12:51:05 -0500 Subject: [PATCH 092/119] #697 Fix tests and update argument order --- fre/tests/test_fre_workflow_cli.py | 101 +++++++++++++++++---- fre/workflow/checkout_script.py | 18 ++-- fre/workflow/freworkflow.py | 10 +- fre/workflow/tests/test_checkout_script.py | 78 ++++++++-------- 4 files changed, 136 insertions(+), 71 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index def167d2d..bd9b1c595 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -1,36 +1,103 @@ -import os -import shutil -from pathlib import Path - -from click.testing import CliRunner - -from fre import fre - """ CLI Tests for fre workflow * Tests the command-line-interface calls for tools in the fre workflow suite. + Each tool generally gets 3 tests: - - fre workflow $tool, checking for exit code 0 (fails if cli isn't configured right) - - fre workflow $tool --help, checking for exit code 0 (fails if the code doesn't run) - - fre workflow $tool --optionDNE, checking for exit code 2 (fails if cli isn't configured - right and thinks the tool has a --optionDNE option) + - fre workflow $tool, checking for exit code 0 (fails if cli isn't configured right) + - fre workflow $tool --help, checking for exit code 0 (fails if the code doesn't run) + - fre workflow $tool --optionDNE, checking for exit code 2; misuse of command (fails if cli isn't configured + right and thinks the tool has a --optionDNE option) """ +import os +from pathlib import Path +from click.testing import CliRunner +from fre import fre runner = CliRunner() -#-- fre pp -def test_cli_fre_workflow(): +## fre workflow subtools search for if TMPDIR is set, specifically for fre workflow checkout --target-dir +# If a value for --target-dir is not passed --> TMPDIR will be used for --target-dir +# If TMPDIR is not set --> a default location will be used for --target-dir +#-- fre workflow +def test_cli_fre_workflow(monkeypatch): ''' fre workflow ''' + monkeypatch.setenv("TMPDIR", "") result = runner.invoke(fre.fre, args=["workflow"]) - assert result.exit_code == 2 + assert result.exit_code == 0 -def test_cli_fre_workflow_help(): +def test_cli_fre_workflow_help(monkeypatch): ''' fre workflow --help ''' + monkeypatch.setenv("TMPDIR", "") result = runner.invoke(fre.fre, args=["workflow", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_opt_dne(): +def test_cli_fre_workflow_opt_dne(monkeypatch): ''' fre workflow optionDNE ''' + monkeypatch.setenv("TMPDIR", "") result = runner.invoke(fre.fre, args=["workflow", "optionDNE"]) assert result.exit_code == 2 +#-- fre workflow checkout +def test_cli_fre_workflow_checkout(monkeypatch): + ''' fre workflow checkout''' + monkeypatch.setenv("TMPDIR", "") + result = runner.invoke(fre.fre, args=["workflow", "checkout"]) + assert result.exit_code == 2 + +def test_cli_fre_workflow_checkout_help(monkeypatch): + ''' fre workflow checkout --help ''' + monkeypatch.setenv("TMPDIR", "") + result = runner.invoke(fre.fre, args=["workflow", "checkout", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_workflow_checkout_opt_dne(monkeypatch): + ''' fre workflow checkout optionDNE ''' + monkeypatch.setenv("TMPDIR", "") + result = runner.invoke(fre.fre, args=["workflow", "checkout", "optionDNE"]) + assert result.exit_code == 2 + +def test_cli_fre_workflow_checkout_target_dir_set(tmp_path): + """ + Test checkout in target directory if --target-dir is explicitly set. + """ + experiment = "c96L65_am5f7b12r1_amip_TESTING" + result = runner.invoke(fre.fre, args=["workflow", "checkout", + "--yamlfile", "fre/workflow/tests/AM5_example/am5.yaml", + "--experiment", experiment, + "--application", "pp", + "--target-dir", tmp_path]) + assert result.exit_code == 0 + assert Path(f"{tmp_path}/cylc-src/{experiment}").exists() + +def test_cli_fre_workflow_checkout_TMPDIR_set(tmp_path, monkeypatch): + """ + Test checkout if TMPDIR environment variable is set and --target-dir has no + specified value + """ + experiment = "c96L65_am5f7b12r1_amip_TESTING" + Path(f"{tmp_path}/env_var").mkdir(parents=True) + monkeypatch.setenv("TMPDIR", f"{tmp_path}/env_var") + + result = runner.invoke(fre.fre, args=["workflow", "checkout", + "-y", "fre/workflow/tests/AM5_example/am5.yaml", + "-e", experiment, + "-a", "pp"]) + assert result.exit_code == 0 + assert Path(f"{os.environ['TMPDIR']}/cylc-src/{experiment}").exists() + +#def test_cli_fre_workflow_checkout_default_dir(): +# """ +# Test checkout if TMPDIR and --target-dir is not set; +# use the default location: ~/.fre +# """ + experiment = "c96L65_am5f7b12r1_amip_TESTING" + monkeypatch.setenv("TMPDIR", "") + + result = runner.invoke(fre.fre, args=["workflow", "checkout", + "-y", "fre/workflow/tests/AM5_example/am5.yaml", + "-e", experiment, + "-a", "pp"]) + #default cylc-src location + default_dir = os.path.expanduser("~/.fre") + assert result.exit_code == 0 + assert Path(f"{default_dir}/cylc-src/{experiment}").exists() diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 35af24d7e..ffdf49812 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -28,8 +28,6 @@ def validate_yaml(yamlfile: dict, application: str): - if gfdl_mdf_schema path is not valid - combined yaml is not valid - unclear error in validation - :return: None - :rtype: None """ schema_dir = Path(__file__).resolve().parents[1] schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json') @@ -74,7 +72,7 @@ def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str): repo, f"{src_dir}/{workflow_name}"], capture_output = True, text = True, check = True) fre_logger.debug(clone_output) - fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + fre_logger.info("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) ## Move combined yaml to cylc-src location current_dir = Path.cwd() @@ -93,7 +91,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N :type experiment: str :param application: Which workflow will be used/cloned :type application: str - :param target_dir: Target directory to clone repository into + :param target_dir: Target/base directory used for cylc-src/ creation :type target_dir: str :param force_checkout: re-clone the workflow repo if it exists :type force_checkout: bool @@ -101,9 +99,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N :raises ValueError: - if the repo and/or tag was not defined - if the target directory does not exist or cannot be found - - if the - raise ValueError('Neither tag nor branch matches the git clone branch arg') - + - if neither tag nor branch matches the git clone branch arg """ # Used in consolidate_yamls function for now platform = None @@ -119,7 +115,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use="run", output=None) - #validate_yaml(yamlfile = yaml, application = "run") + validate_yaml(yamlfile = yaml, application = "run") workflow_info = yaml.get("workflow").get("run") elif application == "pp": # will probably be taken out and put above is "use" @@ -142,11 +138,11 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N fre_logger.warning("(%s):(%s) check out ==> REQUESTED", repo, tag) - # Make sure src_dir exists + # Create src_dir if it does not exist if not Path(target_dir).exists(): - raise ValueError(f"Target directory {target_dir} does not exist or cannot be found.") + Path(target_dir).mkdir(parents=True, exist_ok=True) - # clone directory + # Define cylc-src directory src_dir = f"{target_dir}/cylc-src" # workflow name workflow_name = experiment diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 9b5f87bd1..b31f5e98e 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -27,14 +27,14 @@ def workflow_cli(): @click.option("--target-dir", type=str, envvar="TMPDIR", - default=".fre", - help=f"Target directory for workflow to be cloned into. Default location TMPDIR: {os.environ['TMPDIR']}") + default=os.path.expanduser("~/.fre"), + help=f"Target directory for workflow to be cloned into. TMPDIR will be used if set: {os.environ['TMPDIR']}. If not set, a default location of ") @click.option("--force-checkout", is_flag=True, default=False, - help="If the checkout already, exists, remove and check out again.") -def checkout(yamlfile, experiment, application, target_dir, force_checkout): + help="If the checkout already, exists, remove and clone the desired repo again.") +def checkout(target_dir, yamlfile, experiment, application, force_checkout): """ Checkout/extract fre workflow """ - checkout_script.workflow_checkout(yamlfile, experiment, application, target_dir, force_checkout) + checkout_script.workflow_checkout(target_dir, yamlfile, experiment, application, force_checkout) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index a21c7b5ab..f30b06779 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -7,18 +7,6 @@ TEST_CONFIGS = "fre/workflow/tests/AM5_example/" EXPERIMENT = "c96L65_am5f7b12r1_amip_TESTING" -#@pytest.fixture(autouse=True, name="fake_home") -#def fake_home_fixture(tmp_path, monkeypatch): -# """ -# Set the tmp_path as HOME for the cylc-src directory -# to be created in. -# """ -# ## Mock HOME for cylc-src and cylc-run -# fake_home = Path(tmp_path) -# monkeypatch.setenv("HOME", str(fake_home)) -# -# return fake_home -# def test_cylc_src_creation_fail(tmp_path): """ Test for the expected failure if the cylc-src @@ -40,20 +28,6 @@ def test_cylc_src_creation_fail(tmp_path): application = "pp", target_dir = tmp_path) -def test_checkout_target_dir_dne(tmp_path): - """ - """ - bad_dir = "does/not/exist" - - # run checkout to create cylc-src - expected_error = f"Target directory {bad_dir} does not exist or cannot be found." - - with pytest.raises(ValueError, match = re.escape(expected_error)): - checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", - experiment = EXPERIMENT, - application = "pp", - target_dir = bad_dir) - def test_checkout_missing_repo(tmp_path): """ Test for the expected ValueError if the repo is not @@ -68,14 +42,6 @@ def test_checkout_missing_repo(tmp_path): application = "pp", target_dir = tmp_path) -#def test_run_workflow_checkout(caplog): -# """ -# Test for a successful run workflow checkout. -# """ -# checkout_script.workflow_checkout(yamlfile, -# experiment = "c96L65_am5f7b12r1_amip_TESTING", -# application = "run") - def test_pp_workflow_checkout(tmp_path, caplog): """ Test for a successful post-processing workflow checkout. @@ -90,12 +56,13 @@ def test_pp_workflow_checkout(tmp_path, caplog): assert all([Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").exists(), Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").is_dir(), Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), - Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/config.yaml").exists(), - f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/config.yaml").exists()]) +# f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) def test_pp_workflow_checkout_exists_already(tmp_path, caplog): """ - Test for the expected output message if the checkout/branch already exists. + Test for the expected output message if the checkout already exists, + using the same branch. """ # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", @@ -120,4 +87,39 @@ def test_pp_workflow_checkout_exists_already(tmp_path, caplog): for string in expected_output: assert string in caplog.text -##def test_pp_workflow_checkout_force_checkout +def test_pp_workflow_checkout_force_checkout(tmp_path, caplog): + """ + Test successful re-cloning of the workflow repo if + force-checkout is passed. + """ + # 1st checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path) + + # 2nd checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path, + force_checkout = True) + + src_dir = f"{tmp_path}/cylc-src" + workflow_name = EXPERIMENT + repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + tag = "main" + expected_output = [f" *** PREVIOUS CHECKOUT FOUND: {src_dir}/{workflow_name} *** ", + f" *** REMOVING {src_dir}/{workflow_name} *** "] + # f"({repo}):({tag}) check out ==> SUCCESSFUL"] + + for string in expected_output: + assert string in caplog.text + +#def test_run_workflow_checkout(caplog): +# """ +# Test for a successful run workflow checkout. +# """ +# checkout_script.workflow_checkout(yamlfile, +# experiment = "c96L65_am5f7b12r1_amip_TESTING", +# application = "run") From 8ea6c34a94daed2279f37edd21be0213394a0b8d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 14:01:47 -0500 Subject: [PATCH 093/119] #697 fix exit code --- fre/tests/test_fre_workflow_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index bd9b1c595..54033fcf5 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -23,7 +23,7 @@ def test_cli_fre_workflow(monkeypatch): ''' fre workflow ''' monkeypatch.setenv("TMPDIR", "") result = runner.invoke(fre.fre, args=["workflow"]) - assert result.exit_code == 0 + assert result.exit_code == 2 def test_cli_fre_workflow_help(monkeypatch): ''' fre workflow --help ''' From 67c154496913753b7f3d07278624b1e0a9ffb70d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 14:48:45 -0500 Subject: [PATCH 094/119] #697 Adjust spacing --- fre/tests/test_fre_cmor_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index bf12c33ff..5cdf43360 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -70,8 +70,8 @@ def test_cli_fre_cmor_help_and_debuglog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long - log_text_line_2='[DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_2='[ DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] @@ -89,7 +89,7 @@ def test_cli_fre_cmor_help_and_infolog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] From c619a257b2c3bacffaa098dbf4004090e966344f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 15:03:07 -0500 Subject: [PATCH 095/119] #697 right spacing? --- fre/tests/test_fre_cmor_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index 5cdf43360..c7c08113a 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -70,8 +70,8 @@ def test_cli_fre_cmor_help_and_debuglog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long - log_text_line_2='[ DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_2='[ DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] @@ -89,7 +89,7 @@ def test_cli_fre_cmor_help_and_infolog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] From ca4c2e26c522c6d0123d99598a56de1122005d9d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 15:23:28 -0500 Subject: [PATCH 096/119] #697 set TMPDIR --- fre/tests/test_fre_workflow_cli.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index 54033fcf5..479ad20e1 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -19,40 +19,40 @@ # If a value for --target-dir is not passed --> TMPDIR will be used for --target-dir # If TMPDIR is not set --> a default location will be used for --target-dir #-- fre workflow -def test_cli_fre_workflow(monkeypatch): +def test_cli_fre_workflow(monkeypatch, tmp_path): ''' fre workflow ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow"]) assert result.exit_code == 2 -def test_cli_fre_workflow_help(monkeypatch): +def test_cli_fre_workflow_help(monkeypatch, tmp_path): ''' fre workflow --help ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_opt_dne(monkeypatch): +def test_cli_fre_workflow_opt_dne(monkeypatch, tmp_path): ''' fre workflow optionDNE ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "optionDNE"]) assert result.exit_code == 2 #-- fre workflow checkout -def test_cli_fre_workflow_checkout(monkeypatch): +def test_cli_fre_workflow_checkout(monkeypatch, tmp_path): ''' fre workflow checkout''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout"]) assert result.exit_code == 2 -def test_cli_fre_workflow_checkout_help(monkeypatch): +def test_cli_fre_workflow_checkout_help(monkeypatch, tmp_path): ''' fre workflow checkout --help ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_checkout_opt_dne(monkeypatch): +def test_cli_fre_workflow_checkout_opt_dne(monkeypatch, tmp_path): ''' fre workflow checkout optionDNE ''' - monkeypatch.setenv("TMPDIR", "") + monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout", "optionDNE"]) assert result.exit_code == 2 From 4d19b2f1a9c9e10a9e12d09b60204c79ed82b9ac Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 15:24:09 -0500 Subject: [PATCH 097/119] #697 Change warning to info --- fre/workflow/checkout_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index ffdf49812..15b24654e 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -126,7 +126,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use="pp", output="config.yaml") - #validate_yaml(yamlfile = yaml, application = "pp") + validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp") repo = workflow_info.get("repo") @@ -136,7 +136,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N if None in [repo, tag]: raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}") - fre_logger.warning("(%s):(%s) check out ==> REQUESTED", repo, tag) + fre_logger.info("(%s):(%s) check out ==> REQUESTED", repo, tag) # Create src_dir if it does not exist if not Path(target_dir).exists(): From 3ee11a49370184d58ff2b620fd8fe038533feb49 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 15:53:42 -0500 Subject: [PATCH 098/119] #697 Update help message and other output --- fre/workflow/checkout_script.py | 4 ++-- fre/workflow/freworkflow.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 15b24654e..caf8f0c27 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -44,7 +44,7 @@ def validate_yaml(yamlfile: dict, application: str): # If the yaml is not valid, the schema validation will raise errors and exit try: validate(instance = yamlfile,schema=schema) - fre_logger.info("Combined yaml valid") + fre_logger.info(" ** COMBINED YAML VALID ** ") except SchemaError as exc: raise ValueError(f"Schema '{schema_path}' is not valid. Contact the FRE team.") from exc except ValidationError as exc: @@ -126,7 +126,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use="pp", output="config.yaml") - validate_yaml(yamlfile = yaml, application = "pp") + #validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp") repo = workflow_info.get("repo") diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index b31f5e98e..0e07bda1b 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -28,7 +28,10 @@ def workflow_cli(): type=str, envvar="TMPDIR", default=os.path.expanduser("~/.fre"), - help=f"Target directory for workflow to be cloned into. TMPDIR will be used if set: {os.environ['TMPDIR']}. If not set, a default location of ") + help=f"""Target directory for the workflow to be cloned into. + If not defined, the environment variable TMPDIR will be used, + if available. If both --target-dir and TMPDIR are not set, a + default location of ~/.fre will be used""") @click.option("--force-checkout", is_flag=True, default=False, From da10b19bf8f7847efeb8f2d15b8c06ccfbfb87df Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 25 Feb 2026 16:14:41 -0500 Subject: [PATCH 099/119] #697 Uncomment validation --- fre/workflow/checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index caf8f0c27..437f5410b 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -126,7 +126,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use="pp", output="config.yaml") - #validate_yaml(yamlfile = yaml, application = "pp") + validate_yaml(yamlfile = yaml, application = "pp") workflow_info = yaml.get("workflow").get("pp") repo = workflow_info.get("repo") From 6507c0177c9d249e6c1a11c652a297700896e3d3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Feb 2026 14:06:27 -0500 Subject: [PATCH 100/119] #697 Address ians comments --- fre/workflow/checkout_script.py | 118 +++++++-------------- fre/workflow/tests/test_checkout_script.py | 3 - 2 files changed, 39 insertions(+), 82 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 437f5410b..8c4ce1534 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -52,33 +52,6 @@ def validate_yaml(yamlfile: dict, application: str): except Exception as exc: raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc -def create_checkout(repo: str, tag: str, src_dir: str, workflow_name: str): - """ - Clone the workflow template files from a defined repo into the cylc-src/workflow - directory and move the resolved yaml to the cylc-src directory. - - :param repo: Yaml defined workflow repository - :type repo: str - :param tag: branch or version of defined repository - :type tag: str - :param src_dir: Cylc-src directory - :type src_dir: str - :param workflow_name: Name of workflow - :type workflow_name: src - """ - # scenarios 1+2, checkout doesn't exist, branch specified (or not) - clone_output = subprocess.run( ["git", "clone","--recursive", - f"--branch={tag}", - repo, f"{src_dir}/{workflow_name}"], - capture_output = True, text = True, check = True) - fre_logger.debug(clone_output) - fre_logger.info("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) - - ## Move combined yaml to cylc-src location - current_dir = Path.cwd() - shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") - fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) - def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a defined repo. @@ -105,29 +78,16 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N platform = None target = None - if application == "run": - fre_logger.info("NOT DONE YET") - # will probably be taken out and put above is "use" - # is generalized in this tool + if application in ["run", "pp"]: + fre_logger.info(" ** Configuring the resolved YAML for the %s **", application) yaml = cy.consolidate_yamls(yamlfile=yamlfile, experiment=experiment, platform=platform, target=target, - use="run", - output=None) - validate_yaml(yamlfile = yaml, application = "run") - workflow_info = yaml.get("workflow").get("run") - elif application == "pp": - # will probably be taken out and put above is "use" - # is generalized in this tool - yaml = cy.consolidate_yamls(yamlfile=yamlfile, - experiment=experiment, - platform=platform, - target=target, - use="pp", + use=application, output="config.yaml") - validate_yaml(yamlfile = yaml, application = "pp") - workflow_info = yaml.get("workflow").get("pp") + validate_yaml(yamlfile = yaml, application = application) + workflow_info = yaml.get("workflow").get(application) repo = workflow_info.get("repo") tag = workflow_info.get("version") @@ -154,40 +114,40 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N raise OSError( f"(checkoutScript) directory {src_dir} wasn't able to be created. exit!") from exc - if not Path(f"{src_dir}/{workflow_name}").is_dir(): - fre_logger.info("Workflow does not yet exist; will create now") - create_checkout(repo = repo, - tag = tag, - src_dir = src_dir, - workflow_name = workflow_name) - elif Path(f"{src_dir}/{workflow_name}").is_dir() and force_checkout: + if Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) - # Remove checked out repo - shutil.rmtree(f"{src_dir}/{workflow_name}") - fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) - # Redo checkout - create_checkout(repo = repo, - tag = tag, - src_dir = src_dir, - workflow_name = workflow_name) - elif Path(f"{src_dir}/{workflow_name}").is_dir() and not force_checkout: - fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) - # the repo checkout does exist, scenarios 3 and 4. - with change_directory(f"{src_dir}/{workflow_name}"): - # capture the branch and tag - # if either match git_clone_branch_arg, then success. otherwise, fail. - current_tag = subprocess.run(["git","describe","--tags"], - capture_output = True, - text = True, check = True).stdout.strip() - current_branch = subprocess.run(["git", "branch", "--show-current"], + if force_checkout: + fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) + shutil.rmtree(f"{src_dir}/{workflow_name}") + else: + with change_directory(f"{src_dir}/{workflow_name}"): + # capture the branch and tag + # if either match git_clone_branch_arg, then success. otherwise, fail. + current_tag = subprocess.run(["git","describe","--tags"], capture_output = True, text = True, check = True).stdout.strip() - - if tag in (current_tag, current_branch): - fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) - else: - fre_logger.error( - "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag) - fre_logger.error( - "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) - raise ValueError('Neither tag nor branch matches the git clone branch arg') + current_branch = subprocess.run(["git", "branch", "--show-current"], + capture_output = True, + text = True, check = True).stdout.strip() + + if tag in (current_tag, current_branch): + fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) + else: + fre_logger.error( + "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag) + fre_logger.error( + "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) + raise ValueError('Neither tag nor branch matches the git clone branch arg') + else: + fre_logger.info("Workflow does not exist; will create now") + clone_output = subprocess.run( ["git", "clone","--recursive", + f"--branch={tag}", + repo, f"{src_dir}/{workflow_name}"], + capture_output = True, text = True, check = True) + fre_logger.debug(clone_output) + fre_logger.info("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + + ## Move combined yaml to cylc-src location + current_dir = Path.cwd() + shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") + fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index f30b06779..8456cf97b 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -57,7 +57,6 @@ def test_pp_workflow_checkout(tmp_path, caplog): Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").is_dir(), Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/config.yaml").exists()]) -# f"({expected_repo}):({expected_tag}) check out ==> SUCCESSFUL" in caplog.text]) def test_pp_workflow_checkout_exists_already(tmp_path, caplog): """ @@ -79,7 +78,6 @@ def test_pp_workflow_checkout_exists_already(tmp_path, caplog): repo = "https://github.com/NOAA-GFDL/fre-workflows.git" expected_tag = "main" expected_output = [ - f"({repo}):({expected_tag}) check out ==> REQUESTED", (f"Checkout exists ('{tmp_path}/cylc-src/c96L65_am5f7b12r1_amip_TESTING'), " f"and matches '{expected_tag}'") ] @@ -111,7 +109,6 @@ def test_pp_workflow_checkout_force_checkout(tmp_path, caplog): tag = "main" expected_output = [f" *** PREVIOUS CHECKOUT FOUND: {src_dir}/{workflow_name} *** ", f" *** REMOVING {src_dir}/{workflow_name} *** "] - # f"({repo}):({tag}) check out ==> SUCCESSFUL"] for string in expected_output: assert string in caplog.text From 1a0e8301935ea5ea638d4be8d4dc4870be4a912e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Feb 2026 15:12:19 -0500 Subject: [PATCH 101/119] #697 Update --- fre/workflow/checkout_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 8c4ce1534..8b9f797e0 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -138,7 +138,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') - else: + if not Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.info("Workflow does not exist; will create now") clone_output = subprocess.run( ["git", "clone","--recursive", f"--branch={tag}", From 964e31f633a1d1c90bbc2437ed6fcbc042e4c7e8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 5 Mar 2026 13:51:11 -0500 Subject: [PATCH 102/119] #697 Update yaml workflow addition - remove duplication of "pp" and "run" - put workflow repo and version under associated section that already exists --- fre/workflow/checkout_script.py | 10 ++++++++-- .../tests/AM5_example/yaml_include/settings.yaml | 12 +++--------- .../AM5_example/yaml_include/settings_WRONG.yaml | 12 +++--------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 8b9f797e0..cd84917b1 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -86,8 +86,14 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N target=target, use=application, output="config.yaml") + validate_yaml(yamlfile = yaml, application = application) - workflow_info = yaml.get("workflow").get(application) + + # Reset application for pp to make it discoverable in yaml config + if application == "pp": + application = "postprocess" + + workflow_info = yaml.get(application).get("workflow") repo = workflow_info.get("repo") tag = workflow_info.get("version") @@ -96,7 +102,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N if None in [repo, tag]: raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}") - fre_logger.info("(%s):(%s) check out ==> REQUESTED", repo, tag) + fre_logger.info("(%s):(%s) check out for %s ==> REQUESTED", repo, tag, application) # Create src_dir if it does not exist if not Path(target_dir).exists(): diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml index c45d9c533..4bf3770c4 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -1,12 +1,3 @@ -#workflow repositories -workflow: - run: - repo: "https://github.com/NOAA-GFDL" - version: "tbd" - pp: - repo: "https://github.com/NOAA-GFDL/fre-workflows.git" - version: "main" - #c96_amip_directories: directories: history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] @@ -16,6 +7,9 @@ directories: #c96_amip_postprocess: postprocess: + workflow: + repo: "https://github.com/NOAA-GFDL/fre-workflows.git" + version: "main" settings: history_segment: "P1Y" site: "ppan" diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml index 312a5e7cd..9a102b685 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -1,12 +1,3 @@ -#workflow repositories -workflow: - run: - repo: "https://github.com/NOAA-GFDL" - version: "tbd" - pp: - repo: - version: "main" - #c96_amip_directories: directories: history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] @@ -16,6 +7,9 @@ directories: #c96_amip_postprocess: postprocess: + workflow: + repo: + version: "main" settings: history_segment: "P1Y" site: "ppan" From c348ceb61a3f0fb47f9c82114f99539b708b40e2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 5 Mar 2026 14:01:10 -0500 Subject: [PATCH 103/119] #697 Update readme --- fre/workflow/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 1006fc61e..19915872a 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -15,3 +15,5 @@ fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7 - `-y, --yamlfile [model yaml] (str; required)` - `-e, --experiment [experiment name] (str; required)` - `-a, --application [ run | pp ] (str; required)` + - `--target-dir [target directory where workflow will be cloned] (str; optional; default - $TMPDIR if set; default if TMPDIR not set - ~/.fre` + - `--force-checkout (bool; optional)` From 390d74ddb22e58e7ce6d772d715602fbedbbfdd7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 12:58:05 -0400 Subject: [PATCH 104/119] #697 Some documentation updates --- fre/workflow/checkout_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index cd84917b1..2b8787a1d 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -17,7 +17,7 @@ def validate_yaml(yamlfile: dict, application: str): """ Validate the format of the yaml file based - on the schema.json in gfdl_msd_schemas + on the schema.json held in [gfdl_msd_schemas](https://github.com/NOAA-GFDL/gfdl_msd_schemas). :param yamlfile: Model, settings, pp, and analysis yaml information combined into a dictionary @@ -54,7 +54,7 @@ def validate_yaml(yamlfile: dict, application: str): def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): """ - Create a directory and clone the workflow template files from a defined repo. + Create a directory and clone the workflow template files from a defined repository. :param yamlfile: Model yaml configuration file :type yamlfile: str From 3edada124f826b01a1f8beca1bb6c345d2cc1f0b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 14:12:21 -0400 Subject: [PATCH 105/119] #697 change repo to repository --- fre/workflow/tests/AM5_example/yaml_include/settings.yaml | 2 +- fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml index 4bf3770c4..44e2ab797 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -8,7 +8,7 @@ directories: #c96_amip_postprocess: postprocess: workflow: - repo: "https://github.com/NOAA-GFDL/fre-workflows.git" + repository: "https://github.com/NOAA-GFDL/fre-workflows.git" version: "main" settings: history_segment: "P1Y" diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml index 9a102b685..538be7d28 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -8,7 +8,7 @@ directories: #c96_amip_postprocess: postprocess: workflow: - repo: + repository: version: "main" settings: history_segment: "P1Y" From 8b94b3f26756a7251e4b95f0fc6886dd971422d1 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 14:22:13 -0400 Subject: [PATCH 106/119] #697 change repo to repository --- fre/workflow/checkout_script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 2b8787a1d..ab6b5cac5 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -66,11 +66,11 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N :type application: str :param target_dir: Target/base directory used for cylc-src/ creation :type target_dir: str - :param force_checkout: re-clone the workflow repo if it exists + :param force_checkout: re-clone the workflow repository if it exists :type force_checkout: bool :raises OSError: if the checkout script was not able to be created :raises ValueError: - - if the repo and/or tag was not defined + - if the repository and/or tag was not defined - if the target directory does not exist or cannot be found - if neither tag nor branch matches the git clone branch arg """ @@ -95,7 +95,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N workflow_info = yaml.get(application).get("workflow") - repo = workflow_info.get("repo") + repo = workflow_info.get("repository") tag = workflow_info.get("version") fre_logger.info("Defined tag ==> '%s'", tag) From 338893c39bd266b7b7d8e99fcb3149648241d2f9 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 14:26:02 -0400 Subject: [PATCH 107/119] #697 update test --- .../tests/AM5_example/yaml_include/settings_WRONG.yaml | 2 +- fre/workflow/tests/test_checkout_script.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml index 538be7d28..f620f8028 100644 --- a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -8,7 +8,7 @@ directories: #c96_amip_postprocess: postprocess: workflow: - repository: + repository: "" version: "main" settings: history_segment: "P1Y" diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 8456cf97b..b69f20bad 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -28,15 +28,16 @@ def test_cylc_src_creation_fail(tmp_path): application = "pp", target_dir = tmp_path) -def test_checkout_missing_repo(tmp_path): +def test_checkout_invalid_resolved_yaml(tmp_path): """ - Test for the expected ValueError if the repo is not - defined in the settings.yaml. + Test for the expected error if the repository is not + defined in the settings.yaml and the yamls could not + be combined """ experiment = "c96L65_am5f7b12r1_amip_TESTING_WRONG" repo = None tag = "main" - with pytest.raises(ValueError, match = f"One of these are None: repo / tag = {repo} / {tag}"): + with pytest.raises(ValueError, match = f"Combined yaml is not valid. Please fix the errors and try again."): checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", experiment = experiment, application = "pp", From b1766d32af4c0a581301d619d30934021bef9cf9 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 15:18:59 -0400 Subject: [PATCH 108/119] #697 Change `fre_logger.warning` to `fre_logger.info` --- fre/workflow/checkout_script.py | 2 +- fre/workflow/tests/test_checkout_script.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index ab6b5cac5..a9218aaba 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -121,7 +121,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N f"(checkoutScript) directory {src_dir} wasn't able to be created. exit!") from exc if Path(f"{src_dir}/{workflow_name}").is_dir(): - fre_logger.warning(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) + fre_logger.info(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) if force_checkout: fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) shutil.rmtree(f"{src_dir}/{workflow_name}") diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index b69f20bad..5a4cf07d9 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -108,11 +108,9 @@ def test_pp_workflow_checkout_force_checkout(tmp_path, caplog): workflow_name = EXPERIMENT repo = "https://github.com/NOAA-GFDL/fre-workflows.git" tag = "main" - expected_output = [f" *** PREVIOUS CHECKOUT FOUND: {src_dir}/{workflow_name} *** ", - f" *** REMOVING {src_dir}/{workflow_name} *** "] + expected_output = f" *** REMOVING {src_dir}/{workflow_name} *** " - for string in expected_output: - assert string in caplog.text + assert expected_output in caplog.text #def test_run_workflow_checkout(caplog): # """ From dfe8c38faaa3f50a9a5ac4957057bb7cc3de82e8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 17:05:32 -0400 Subject: [PATCH 109/119] #697 Remove envvar TMPDIR usage as default for now --- fre/workflow/README.md | 2 +- fre/workflow/freworkflow.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 19915872a..c0759c2a5 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -15,5 +15,5 @@ fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7 - `-y, --yamlfile [model yaml] (str; required)` - `-e, --experiment [experiment name] (str; required)` - `-a, --application [ run | pp ] (str; required)` - - `--target-dir [target directory where workflow will be cloned] (str; optional; default - $TMPDIR if set; default if TMPDIR not set - ~/.fre` + - `--target-dir [target directory where workflow will be cloned] (str; optional; default is ~/.fre-workflows` - `--force-checkout (bool; optional)` diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 0e07bda1b..be8c6a4ea 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -26,12 +26,10 @@ def workflow_cli(): required=True) @click.option("--target-dir", type=str, - envvar="TMPDIR", - default=os.path.expanduser("~/.fre"), + default=os.path.expanduser("~/.fre-workflows"), help=f"""Target directory for the workflow to be cloned into. - If not defined, the environment variable TMPDIR will be used, - if available. If both --target-dir and TMPDIR are not set, a - default location of ~/.fre will be used""") + If not defined, a default location of ~/.fre-workflows + will be used""") @click.option("--force-checkout", is_flag=True, default=False, From e7e9f5fa0f161f66de6ba245e1d48a791a28be4c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 9 Mar 2026 17:07:33 -0400 Subject: [PATCH 110/119] #697 update doc --- fre/workflow/freworkflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index be8c6a4ea..29bdfc97e 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -1,4 +1,4 @@ -''' fre workflow ''' +''' fre workflow click interface for fre workflow subcommands''' import os import click import logging From 9aab4447d33706c941722316cd8d3cd6c6348ad9 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 16 Mar 2026 11:45:50 -0400 Subject: [PATCH 111/119] #697 Address some of Mikyung's comments --- fre/workflow/checkout_script.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index a9218aaba..95ceb6917 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -16,17 +16,17 @@ ######VALIDATE##### def validate_yaml(yamlfile: dict, application: str): """ - Validate the format of the yaml file based - on the schema.json held in [gfdl_msd_schemas](https://github.com/NOAA-GFDL/gfdl_msd_schemas). + Validate the format of the yaml file against the + schema.json held in [gfdl_msd_schemas](https://github.com/NOAA-GFDL/gfdl_msd_schemas). - :param yamlfile: Model, settings, pp, and analysis yaml - information combined into a dictionary + :param yamlfile: Dictionary containing the combined model, + settings, pp, and analysis yaml content :type yamlfile: dict :param application: type of workflow to check out/clone :type application: string :raises ValueError: - - if gfdl_mdf_schema path is not valid - - combined yaml is not valid + - invalid gfdl_msd_schema path + - invalid combined yaml - unclear error in validation """ schema_dir = Path(__file__).resolve().parents[1] @@ -54,25 +54,26 @@ def validate_yaml(yamlfile: dict, application: str): def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): """ - Create a directory and clone the workflow template files from a defined repository. + Create a directory and clone the workflow template files from a specified repository. :param yamlfile: Model yaml configuration file :type yamlfile: str - :param experiment: One of the postprocessing experiment names from the - yaml displayed by fre list exps -y $yamlfile - (e.g. c96L65_am5f4b4r0_amip), default None + :param experiment: One of the experiment names listed in the model yaml file. + Note: the command "fre list exps -y [model_yamlfile]" can be used to + list the available experiment names :type experiment: str - :param application: Which workflow will be used/cloned + :param application: String used to specify the type of workflow to be used/cloned. + Ex.: run, postprocess :type application: str - :param target_dir: Target/base directory used for cylc-src/ creation + :param target_dir: Target location to create the cylc-src/ directory in :type target_dir: str :param force_checkout: re-clone the workflow repository if it exists :type force_checkout: bool - :raises OSError: if the checkout script was not able to be created + :raises OSError: if the checkout script cannot be created :raises ValueError: - if the repository and/or tag was not defined - if the target directory does not exist or cannot be found - - if neither tag nor branch matches the git clone branch arg + - if tag or branch does not match the git clone branch arg """ # Used in consolidate_yamls function for now platform = None From a8eed8a35666d6c8d609f38709bbf78e8a4953d2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 16 Mar 2026 11:57:13 -0400 Subject: [PATCH 112/119] #697 addressing comments pt. 2 --- fre/tests/test_fre_workflow_cli.py | 13 ++++--------- fre/workflow/tests/test_checkout_script.py | 8 ++++---- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index 479ad20e1..b2845c910 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -1,12 +1,10 @@ """ CLI Tests for fre workflow * -Tests the command-line-interface calls for tools in the fre workflow suite. -Each tool generally gets 3 tests: - - fre workflow $tool, checking for exit code 0 (fails if cli isn't configured right) - - fre workflow $tool --help, checking for exit code 0 (fails if the code doesn't run) - - fre workflow $tool --optionDNE, checking for exit code 2; misuse of command (fails if cli isn't configured - right and thinks the tool has a --optionDNE option) +Tests the command-line-interface commands for each tool in the fre workflow suite. + - successful invocation of fre workflow $tool + - successful invocation of fre workflow $tool --help + - expected failure for fre workflow $tool --optionDne (failure for no click option defined for what was passed) """ import os from pathlib import Path @@ -15,9 +13,6 @@ runner = CliRunner() -## fre workflow subtools search for if TMPDIR is set, specifically for fre workflow checkout --target-dir -# If a value for --target-dir is not passed --> TMPDIR will be used for --target-dir -# If TMPDIR is not set --> a default location will be used for --target-dir #-- fre workflow def test_cli_fre_workflow(monkeypatch, tmp_path): ''' fre workflow ''' diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 5a4cf07d9..135ae1aaa 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -12,8 +12,8 @@ def test_cylc_src_creation_fail(tmp_path): Test for the expected failure if the cylc-src directory cannot be created. - This test simulates a file with the name cylc-src - already created, causing a permission error in HOME. + This test simulates the instance where a file with the name + 'cylc-src' already exists, causing a permission error in HOME. """ cylc_src_file = Path(f"{tmp_path}/cylc-src") with open(cylc_src_file, "w", encoding='utf-8') as f: @@ -88,8 +88,8 @@ def test_pp_workflow_checkout_exists_already(tmp_path, caplog): def test_pp_workflow_checkout_force_checkout(tmp_path, caplog): """ - Test successful re-cloning of the workflow repo if - force-checkout is passed. + Test successful re-cloning of the workflow repo + when force-checkout=True. """ # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", From b1b5142a322c322f649d26d41310f3f5485b5e40 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 16 Mar 2026 17:58:06 -0400 Subject: [PATCH 113/119] #697 address comments pt 3 --- fre/tests/test_fre_workflow_cli.py | 2 +- fre/workflow/tests/test_checkout_script.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index b2845c910..90559f515 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -4,7 +4,7 @@ Tests the command-line-interface commands for each tool in the fre workflow suite. - successful invocation of fre workflow $tool - successful invocation of fre workflow $tool --help - - expected failure for fre workflow $tool --optionDne (failure for no click option defined for what was passed) + - expected failure for fre workflow $tool --optionDne (failure for undefined click option) """ import os from pathlib import Path diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py index 135ae1aaa..34d0a39b0 100644 --- a/fre/workflow/tests/test_checkout_script.py +++ b/fre/workflow/tests/test_checkout_script.py @@ -31,7 +31,7 @@ def test_cylc_src_creation_fail(tmp_path): def test_checkout_invalid_resolved_yaml(tmp_path): """ Test for the expected error if the repository is not - defined in the settings.yaml and the yamls could not + defined in the settings.yaml and the yamls cannot be combined """ experiment = "c96L65_am5f7b12r1_amip_TESTING_WRONG" @@ -61,8 +61,7 @@ def test_pp_workflow_checkout(tmp_path, caplog): def test_pp_workflow_checkout_exists_already(tmp_path, caplog): """ - Test for the expected output message if the checkout already exists, - using the same branch. + Test for the expected output message if the checkout already exists. """ # 1st checkout checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", From 7cffb661ed619601b448c935abde0ea3cd9426a0 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 19 Mar 2026 15:09:58 -0400 Subject: [PATCH 114/119] #697 update doc --- fre/workflow/freworkflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 29bdfc97e..11cace332 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -36,6 +36,6 @@ def workflow_cli(): help="If the checkout already, exists, remove and clone the desired repo again.") def checkout(target_dir, yamlfile, experiment, application, force_checkout): """ - Checkout/extract fre workflow + Checkout/clone the workflow repository. """ checkout_script.workflow_checkout(target_dir, yamlfile, experiment, application, force_checkout) From 0230fd9d838625dec7538a80652e1120a5145ed9 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 14:20:53 -0400 Subject: [PATCH 115/119] #697 fix wording --- fre/workflow/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index c0759c2a5..270f85862 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -1,19 +1,20 @@ # FRE workflow -`fre workflow` provides subtools that help to clone, install, and run a workflow from a repository. + +The`fre workflow` toolset allows user to clone, install, and run a cylc workflow. ## Quickstart -From the root of the fre-cli repository, run: +From the top-level dircetory of the fre-cli repository: ``` # Checkout/clone the post-processing workflow repository -fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING -a pp +fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING --application pp ``` ## Subtools - `fre workflow checkout [options]` - - Purpose: Clone the workflow repository/branch, depending on the application passed. + - Purpose: Clone the specified workflow repository from the settings.yaml, associated with the application passed. - Options: - `-y, --yamlfile [model yaml] (str; required)` - `-e, --experiment [experiment name] (str; required)` - `-a, --application [ run | pp ] (str; required)` - - `--target-dir [target directory where workflow will be cloned] (str; optional; default is ~/.fre-workflows` + - `--target-dir [target location where workflow will be cloned] (str; optional; default is ~/.fre-workflows` - `--force-checkout (bool; optional)` From 1e29e4e08a8b235be8d789b061ea4fcfccc6eb79 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 16:08:25 -0400 Subject: [PATCH 116/119] #697 Fix tests - define target_dir default in script, not in argument/click option --- fre/tests/test_fre_workflow_cli.py | 50 +++++++++--------------------- fre/workflow/freworkflow.py | 1 - 2 files changed, 14 insertions(+), 37 deletions(-) diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py index 90559f515..db2a002ae 100644 --- a/fre/tests/test_fre_workflow_cli.py +++ b/fre/tests/test_fre_workflow_cli.py @@ -6,7 +6,6 @@ - successful invocation of fre workflow $tool --help - expected failure for fre workflow $tool --optionDne (failure for undefined click option) """ -import os from pathlib import Path from click.testing import CliRunner from fre import fre @@ -14,40 +13,34 @@ runner = CliRunner() #-- fre workflow -def test_cli_fre_workflow(monkeypatch, tmp_path): +def test_cli_fre_workflow(): ''' fre workflow ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow"]) assert result.exit_code == 2 -def test_cli_fre_workflow_help(monkeypatch, tmp_path): +def test_cli_fre_workflow_help(): ''' fre workflow --help ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_opt_dne(monkeypatch, tmp_path): +def test_cli_fre_workflow_opt_dne(): ''' fre workflow optionDNE ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "optionDNE"]) assert result.exit_code == 2 #-- fre workflow checkout -def test_cli_fre_workflow_checkout(monkeypatch, tmp_path): +def test_cli_fre_workflow_checkout(): ''' fre workflow checkout''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout"]) assert result.exit_code == 2 -def test_cli_fre_workflow_checkout_help(monkeypatch, tmp_path): +def test_cli_fre_workflow_checkout_help(): ''' fre workflow checkout --help ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout", "--help"]) assert result.exit_code == 0 -def test_cli_fre_workflow_checkout_opt_dne(monkeypatch, tmp_path): +def test_cli_fre_workflow_checkout_opt_dne(): ''' fre workflow checkout optionDNE ''' - monkeypatch.setenv("TMPDIR", tmp_path) result = runner.invoke(fre.fre, args=["workflow", "checkout", "optionDNE"]) assert result.exit_code == 2 @@ -64,35 +57,20 @@ def test_cli_fre_workflow_checkout_target_dir_set(tmp_path): assert result.exit_code == 0 assert Path(f"{tmp_path}/cylc-src/{experiment}").exists() -def test_cli_fre_workflow_checkout_TMPDIR_set(tmp_path, monkeypatch): +def test_cli_fre_workflow_checkout_default_dir(monkeypatch, tmp_path): """ - Test checkout if TMPDIR environment variable is set and --target-dir has no - specified value + Test workflow repository is cloned in the default location + if --target-dir is not set; default = ~./fre-workflows """ - experiment = "c96L65_am5f7b12r1_amip_TESTING" - Path(f"{tmp_path}/env_var").mkdir(parents=True) - monkeypatch.setenv("TMPDIR", f"{tmp_path}/env_var") - - result = runner.invoke(fre.fre, args=["workflow", "checkout", - "-y", "fre/workflow/tests/AM5_example/am5.yaml", - "-e", experiment, - "-a", "pp"]) - assert result.exit_code == 0 - assert Path(f"{os.environ['TMPDIR']}/cylc-src/{experiment}").exists() + # Create and set a mock HOME + fake_home = f"{tmp_path}/fake_home" + Path(fake_home).mkdir(parents=True,exist_ok=True) + monkeypatch.setenv("HOME", f"{tmp_path}/fake_home") -#def test_cli_fre_workflow_checkout_default_dir(): -# """ -# Test checkout if TMPDIR and --target-dir is not set; -# use the default location: ~/.fre -# """ experiment = "c96L65_am5f7b12r1_amip_TESTING" - monkeypatch.setenv("TMPDIR", "") - result = runner.invoke(fre.fre, args=["workflow", "checkout", "-y", "fre/workflow/tests/AM5_example/am5.yaml", "-e", experiment, "-a", "pp"]) - #default cylc-src location - default_dir = os.path.expanduser("~/.fre") assert result.exit_code == 0 - assert Path(f"{default_dir}/cylc-src/{experiment}").exists() + assert Path(f"{fake_home}/.fre-workflows/cylc-src/{experiment}").exists() diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 11cace332..e2b485031 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -26,7 +26,6 @@ def workflow_cli(): required=True) @click.option("--target-dir", type=str, - default=os.path.expanduser("~/.fre-workflows"), help=f"""Target directory for the workflow to be cloned into. If not defined, a default location of ~/.fre-workflows will be used""") From be774dc52a80d62f9f98464591963ac816590ac7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 16:33:42 -0400 Subject: [PATCH 117/119] #697 archive the workflow directory instead, address comments --- fre/workflow/checkout_script.py | 58 +++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 95ceb6917..6c60767c3 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -1,12 +1,14 @@ """ Workflow checkout """ import os import subprocess +import filecmp from pathlib import Path import logging import shutil +from datetime import datetime +from typing import Optional import json from jsonschema import validate, SchemaError, ValidationError -from typing import Optional import fre.yamltools.combine_yamls_script as cy from fre.app.helpers import change_directory @@ -27,7 +29,7 @@ def validate_yaml(yamlfile: dict, application: str): :raises ValueError: - invalid gfdl_msd_schema path - invalid combined yaml - - unclear error in validation + - miscellaneous error in validation """ schema_dir = Path(__file__).resolve().parents[1] schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json') @@ -50,9 +52,10 @@ def validate_yaml(yamlfile: dict, application: str): except ValidationError as exc: raise ValueError("Combined yaml is not valid. Please fix the errors and try again.") from exc except Exception as exc: - raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc + raise ValueError("Miscellaneous error from validation. Please try to find the error and try again.") from exc -def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False): +def workflow_checkout(target_dir: str = None, yamlfile: str = None, experiment: str = None, + application: str = None, force_checkout: Optional[bool] = False): """ Create a directory and clone the workflow template files from a specified repository. @@ -79,6 +82,10 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N platform = None target = None + # Set the default target directory location + if target_dir is None: + target_dir = os.path.expanduser("~/.fre-workflows") + if application in ["run", "pp"]: fre_logger.info(" ** Configuring the resolved YAML for the %s **", application) yaml = cy.consolidate_yamls(yamlfile=yamlfile, @@ -96,6 +103,7 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N workflow_info = yaml.get(application).get("workflow") + yaml_filepath = f"{Path.cwd()}/config.yaml" repo = workflow_info.get("repository") tag = workflow_info.get("version") fre_logger.info("Defined tag ==> '%s'", tag) @@ -124,10 +132,24 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N if Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.info(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) if force_checkout: - fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) - shutil.rmtree(f"{src_dir}/{workflow_name}") + # Create archived workflows location + archived = f"{target_dir}/archived_workflows" + Path(archived).mkdir(parents=True, exist_ok=True) + + # Move previous workflow to archived location + fre_logger.warning(" *** Moving previous checkout to %s ***", archived) + shutil.move(f"{src_dir}/{workflow_name}", archived) + + # Rename previous workflow + move_timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + os.rename(f"{archived}/{workflow_name}", f"{archived}/{workflow_name}_{move_timestamp}") + +# # Keeping this here in case we want to switch force_checkout to removing instead of moving +# fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) +# shutil.rmtree(f"{src_dir}/{workflow_name}") else: with change_directory(f"{src_dir}/{workflow_name}"): + ## Compare previous workflow directory # capture the branch and tag # if either match git_clone_branch_arg, then success. otherwise, fail. current_tag = subprocess.run(["git","describe","--tags"], @@ -138,13 +160,35 @@ def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = N text = True, check = True).stdout.strip() if tag in (current_tag, current_branch): - fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) + fre_logger.info("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) else: fre_logger.error( "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag) fre_logger.error( "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) raise ValueError('Neither tag nor branch matches the git clone branch arg') + + + ## Compare content of current and previous configured, resolved yamls + if filecmp.cmp(yaml_filepath, "config.yaml", shallow=False): + fre_logger.info("Resolved yaml already exists and did not change.") + else: + fre_logger.error("") + fre_logger.error("ERROR: Checkout and resolved yaml already exist but resolved yaml files " + "are not identical!") + fre_logger.error("For troubleshooting:") + fre_logger.error(" - Current resolved yaml: %s", yaml_filepath) + fre_logger.error(" - Previous resolved yaml: %s", f"{src_dir}/{workflow_name}/config.yaml") + fre_logger.error("Try:") + fre_logger.error(" - resolving yaml differences if nothing else has changed") + fre_logger.error(f" - removing the {target_dir}/cylc-src/{workflow_name} folder and " + "re-running the command") + fre_logger.error(" - pass the --force-checkout option to archive the workflow (move to " + "~/.fre-workflows/archived/) and clone a new workflow.") + return +# raise ValueError("Resolve yaml differences or pass --force-checkout to archive the workflow" +# "(moved to ~/.fre-workflows/archived) and clone a new workflow.") + if not Path(f"{src_dir}/{workflow_name}").is_dir(): fre_logger.info("Workflow does not exist; will create now") clone_output = subprocess.run( ["git", "clone","--recursive", From 7cf7523dc3bf7264d8ebc1fcce441481bbf48643 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 16:51:42 -0400 Subject: [PATCH 118/119] #697 update doc --- fre/workflow/checkout_script.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 6c60767c3..6d025f8bc 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -70,7 +70,8 @@ def workflow_checkout(target_dir: str = None, yamlfile: str = None, experiment: :type application: str :param target_dir: Target location to create the cylc-src/ directory in :type target_dir: str - :param force_checkout: re-clone the workflow repository if it exists + :param force_checkout: If the workflow directory exists, move it to an archived location + (~/.fre-workflows/archived) and re-clone the workflow repository :type force_checkout: bool :raises OSError: if the checkout script cannot be created :raises ValueError: From 4a94f606ec55f1c0e487b4ac7919de8914d925c8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 20 Mar 2026 16:59:20 -0400 Subject: [PATCH 119/119] #697 update doc --- fre/workflow/README.md | 4 +++- fre/workflow/checkout_script.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fre/workflow/README.md b/fre/workflow/README.md index 270f85862..75bc72dc3 100644 --- a/fre/workflow/README.md +++ b/fre/workflow/README.md @@ -1,6 +1,8 @@ # FRE workflow -The`fre workflow` toolset allows user to clone, install, and run a cylc workflow. +The`fre workflow` toolset allows users to clone, install, and run a cylc workflow. + +The workflow repository and version are specified in `the setting.yaml`. ## Quickstart From the top-level dircetory of the fre-cli repository: diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py index 6d025f8bc..744c4bc0f 100644 --- a/fre/workflow/checkout_script.py +++ b/fre/workflow/checkout_script.py @@ -65,7 +65,7 @@ def workflow_checkout(target_dir: str = None, yamlfile: str = None, experiment: Note: the command "fre list exps -y [model_yamlfile]" can be used to list the available experiment names :type experiment: str - :param application: String used to specify the type of workflow to be used/cloned. + :param application: String used to specify the type of workflow to be cloned. Ex.: run, postprocess :type application: str :param target_dir: Target location to create the cylc-src/ directory in