diff --git a/docs/tools/listtools.rst b/docs/tools/listtools.rst index 0501b1b24..05edb6990 100644 --- a/docs/tools/listtools.rst +++ b/docs/tools/listtools.rst @@ -2,7 +2,7 @@ ----------------- ``fre list exps [options]`` - - Purpose: Lists available post-processing experiments included in the yaml configurations + - Purpose: List available post-processing experiments included in the yaml configurations - Options: - ``-y, --yamlfile [experiment yaml]`` @@ -10,7 +10,7 @@ ----------------- ``fre list platforms [options]`` - - Purpose: Lists available platforms included in the yaml configurations + - Purpose: List available platforms included in the yaml configurations - Options: - ``-y, --yamlfile [experiment yaml]`` @@ -18,7 +18,7 @@ ----------------- ``fre list pp-components [options]`` - - Purpose: Lists components that have the `postprocess_on` key set to `True` in the postprocessing yaml configurations + - Purpose: List components in the postprocessing yaml that will be post-processed. All components will be post-processed unless ``postprocess_on: False`` is specified - Options: - ``-y, --yamlfile [experiment yaml]`` - ``-e, --experiment [experiment to be post-processed]`` diff --git a/docs/usage/postprocess.rst b/docs/usage/postprocess.rst index 4989d735d..581eeba59 100644 --- a/docs/usage/postprocess.rst +++ b/docs/usage/postprocess.rst @@ -79,9 +79,9 @@ Required configuration postprocess: settings: - pp_start: 1979-01-01T0000Z + pp_start: "1979-01-01T0000Z" - pp_stop: 2020-01-01T0000Z + pp_stop: "2020-01-01T0000Z" Postprocess components ---------------------- diff --git a/docs/usage/yaml_dev/pp_yaml.rst b/docs/usage/yaml_dev/pp_yaml.rst index 6c76e15b2..95ba5fbf2 100644 --- a/docs/usage/yaml_dev/pp_yaml.rst +++ b/docs/usage/yaml_dev/pp_yaml.rst @@ -27,7 +27,6 @@ Required keys include: - type - sources - - postprocess_on **Settings yaml** @@ -52,11 +51,8 @@ This file can follow the format below: pp_chunks: "array of ISO8601 datetime durations, specifying the interval of simulated time per postprocessed file" (string) pp_grid_spec: "path to FMS grid definition tarfile" (string) switches: - do_timeavgs: "switch to turn on/off time-average file generation" (boolean) clean_work: "switch to remove intermediate data files when they are no longer needed" (boolean) do_atmos_plevel_masking: "switch to mask atmos pressure-level output above/below surface pressure/atmos top" (boolean) - do_analysis: "switch to launch analysis scripts" (boolean) - do_analysis_only: "switch to only launch analysis scripts" (boolean) Required keys include: @@ -72,5 +68,3 @@ Required keys include: - clean_work - do_timeavgs - do_atmos_plevel_masking - - do_analysis - - do_analysis_only diff --git a/fre/app/regrid_xy/regrid_xy.py b/fre/app/regrid_xy/regrid_xy.py index f70d6e472..76b13696b 100644 --- a/fre/app/regrid_xy/regrid_xy.py +++ b/fre/app/regrid_xy/regrid_xy.py @@ -342,12 +342,14 @@ def regrid_xy(yamlfile: str, # submit fregrid job for each component for component in components: - - # skip component if postprocess_on = False - if not component["postprocess_on"]: - fre_logger.warning(f"postprocess_on=False for {source} in component {component['type']}." \ - "Skipping {source}") - continue + # If postprocess_on is not defined, it should have the default value of True + # If postprocess_on is defined, check for a True or False value + if "postprocess_on" in component: + # skip component if postprocess_on = False + if not component["postprocess_on"]: + fre_logger.warning(f"postprocess_on=False for {source} in component {component['type']}." \ + "Skipping {source}") + continue # skip component if xyInterp is not set if 'xyInterp' not in component: diff --git a/fre/gfdl_msd_schemas b/fre/gfdl_msd_schemas index 595f1b39a..ee6a34da8 160000 --- a/fre/gfdl_msd_schemas +++ b/fre/gfdl_msd_schemas @@ -1 +1 @@ -Subproject commit 595f1b39acf7604e6edb9c72d2221c59e69529d2 +Subproject commit ee6a34da83d89988c6813b24f87807099a572aee diff --git a/fre/list_/list_pp_components_script.py b/fre/list_/list_pp_components_script.py index 7965a02d7..c424a033a 100644 --- a/fre/list_/list_pp_components_script.py +++ b/fre/list_/list_pp_components_script.py @@ -44,7 +44,10 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str): # log the experiment names, which should show up on screen for sure fre_logger.info("Components to be post-processed:") for i in yml_dict["postprocess"]["components"]: - if i.get("postprocess_on"): + if "postprocess_on" in i: + if i.get("postprocess_on") is True: + fre_logger.info(' - %s', i.get("type")) + else: fre_logger.info(' - %s', i.get("type")) fre_logger.info("\n") diff --git a/fre/list_/tests/test_list_pp_components_script.py b/fre/list_/tests/test_list_pp_components_script.py index 7372ea645..cb9bf96a4 100644 --- a/fre/list_/tests/test_list_pp_components_script.py +++ b/fre/list_/tests/test_list_pp_components_script.py @@ -2,13 +2,7 @@ Test fre list pp-comps """ from pathlib import Path - -import pytest -import yaml - from fre.list_ import list_pp_components_script -from fre.yamltools import combine_yamls_script as cy -from fre.yamltools import helpers # SET-UP @@ -37,13 +31,18 @@ def test_ppyamls_exist(): assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{pp_yaml}").exists() # Test whole tool -def test_exp_list(caplog): +def test_pp_comp_list(caplog): ''' Test fre list pp-components subtool ''' list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME) # check the logging output check_out = [ 'Components to be post-processed:', + ' - atmos_cmip', ' - atmos', + ' - atmos_level_cmip', + ' - atmos_level', + ' - atmos_month_aer', + ' - atmos_diurnal', ' - atmos_scalar'] for i in check_out: @@ -66,4 +65,4 @@ def test_yamlvalidate(caplog): assert i in caplog.text for record in caplog.records: - record.levelname == "INFO" + assert record.levelname == "INFO" diff --git a/fre/make/tests/null_example/settings.yaml b/fre/make/tests/null_example/settings.yaml index dbac1dc9e..b41b83a82 100644 --- a/fre/make/tests/null_example/settings.yaml +++ b/fre/make/tests/null_example/settings.yaml @@ -9,5 +9,3 @@ postprocess: site: "ppan" switches: clean_work: True - do_refinediag: False - do_analysis: True diff --git a/fre/make/tests/null_example/wrong_model/wrong_null_model.yaml b/fre/make/tests/null_example/wrong_model/wrong_null_model.yaml index de0a3cefd..22d6ea9a6 100644 --- a/fre/make/tests/null_example/wrong_model/wrong_null_model.yaml +++ b/fre/make/tests/null_example/wrong_model/wrong_null_model.yaml @@ -10,28 +10,3 @@ fre_properties: build: compileYaml: "../compile.yaml" platformYaml: "empty_platforms.yaml" - -shared: # directories shared across tools - directories: &shared_directories - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] - - # shared pp settings - postprocess: - settings: &shared_settings - site: "ppan" - switches: &shared_switches - clean_work: True - do_refinediag: False - do_analysis: True - -experiments: - - name: "null_model_full" - pp: - - name: "null_model_0" - pp: - - name: "null_model_1" - pp: - - name: "null_model_2" - pp: diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index a6fec203b..2dbe63cdf 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -122,6 +122,7 @@ def set_rose_suite(yamlfile: dict, rose_suite: metomi.rose.config.ConfigNode) -> """ pp=yamlfile.get("postprocess") dirs=yamlfile.get("directories") + analysis=yamlfile.get("analysis") # set rose-suite items pa_scripts = "" @@ -186,6 +187,34 @@ def set_rose_suite(yamlfile: dict, rose_suite: metomi.rose.config.ConfigNode) -> rose_suite.set( keys = ['template variables', 'DO_PREANALYSIS'], value = 'False' ) + # Set DO_ANALYSIS switch + # analysis_on is optional key for each component in the analysis yaml and + # defaults to True if not specified. + # In the rose_suite.conf: + # - if 'analysis_on: False' for all analysis components, set DO_ANALYSIS=False + # - if 'analysis_on: True' for any analysis components, set DO_ANALYSIS=True + do_analysis_switch = [] + if not analysis: + return + + for an_key, an_value in analysis.items(): + an_workflow_info = an_value["workflow"] + # if analysis_on key is actually set, evaluate and save its value in a list + if "analysis_on" in an_workflow_info: + do_analysis_switch.append(an_workflow_info["analysis_on"]) + #if analysis_on key is NOT set, save its value as True in the list + else: + do_analysis_switch.append("True") + + # if ANY of the analysis components do not set analysis_on or set analysis_on: True, + # set DO_ANALYSIS=True in the rose_suite.conf + if any(do_analysis_switch): + rose_suite.set( keys = ['template variables', 'DO_ANALYSIS'], + value = 'True' ) + else: + rose_suite.set( keys = ['template variables', 'DO_ANALYSIS'], + value = 'False' ) + if dirs is not None: for key,value in dirs.items(): rose_suite.set(keys=['template variables', key.upper()], value=quote_rose_values(value)) diff --git a/fre/pp/tests/AM5_example/am5.yaml b/fre/pp/tests/AM5_example/am5.yaml index 960863583..24d365b25 100644 --- a/fre/pp/tests/AM5_example/am5.yaml +++ b/fre/pp/tests/AM5_example/am5.yaml @@ -6,8 +6,8 @@ fre_properties: # amip - &EXP_AMIP_START "19790101T0000Z" - &EXP_AMIP_END "20200101T0000Z" - - &ANA_AMIP_START "19800101T0000Z" - - &ANA_AMIP_END "20200101T0000Z" + - &ANA_AMIP_START "1980" + - &ANA_AMIP_END "2020" - &PP_AMIP_CHUNK96 "P1Y" - &PP_AMIP_CHUNK384 "P1Y" diff --git a/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml index d20d05757..3859217bd 100644 --- a/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml +++ b/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml @@ -15,7 +15,6 @@ postprocess: xyInterp: *custom_interp interpMethod: "conserve_order2" inputRealm: 'atmos' - postprocess_on: False - type: "atmos" sources: - history_file: "atmos_month" @@ -23,7 +22,6 @@ postprocess: xyInterp: *PP_XYINTERP96 interpMethod: "conserve_order2" inputRealm: 'atmos' - postprocess_on: True - type: "atmos_level_cmip" sources: - history_file: "atmos_level_cmip" @@ -31,7 +29,6 @@ postprocess: xyInterp: *PP_XYINTERP96 interpMethod: "conserve_order2" inputRealm: 'atmos' - postprocess_on: False - type: "atmos_level" sources: - history_file: "atmos_month" @@ -39,7 +36,6 @@ postprocess: xyInterp: *PP_XYINTERP96 interpMethod: "conserve_order2" inputRealm: 'atmos' - postprocess_on: False - type: "atmos_month_aer" sources: - history_file: "atmos_month_aer" @@ -47,7 +43,6 @@ postprocess: xyInterp: *PP_XYINTERP96 interpMethod: "conserve_order1" inputRealm: 'atmos' - postprocess_on: False - type: "atmos_diurnal" sources: - history_file: "atmos_diurnal" @@ -55,11 +50,9 @@ postprocess: xyInterp: *PP_XYINTERP96 interpMethod: "conserve_order2" inputRealm: 'atmos' - postprocess_on: False - type: "atmos_scalar" sources: - history_file: "atmos_scalar" - postprocess_on: True - type: "aerosol_cmip" xyInterp: *PP_XYINTERP96 sources: diff --git a/fre/pp/tests/AM5_example/yaml_include/settings.yaml b/fre/pp/tests/AM5_example/yaml_include/settings.yaml index 4d0325549..1057fc6d5 100644 --- a/fre/pp/tests/AM5_example/yaml_include/settings.yaml +++ b/fre/pp/tests/AM5_example/yaml_include/settings.yaml @@ -19,8 +19,6 @@ postprocess: switches: &shared_switches clean_work: True do_atmos_plevel_masking: True - do_analysis: True - do_analysis_only: False preanalysis: vitals: script: "/path/to/vitals-script" diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py index b1ecf6815..870e727b2 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -159,6 +159,31 @@ def test_cleanup(): shutil.rmtree(f"{TEST_DIR}/configure_yaml_out") assert not Path(f"{TEST_DIR}/configure_yaml_out").exists() +def test_rose_suite_DO_ANALYSIS(): + """ + """ + rose_suite = metomi.rose.config.ConfigNode() + yaml_dict = { + "postprocess": {"settings": {"some_setting": "value"}}, + "directories": {"pp_dir": "/some/path"}, + "analysis": { + "land-test": { + "required": { + "data_frequency": "mon", + "date_range": ["19800101T0000Z", "20200101T0000Z"] + }, + "workflow": { + "components": ["land-test"], + "script_type": "one-shot", + "product": "ts", + "chunk_size": "P1Y" + } + } + } + } + csy.set_rose_suite(yaml_dict, rose_suite) + assert rose_suite.get(['template variables', 'DO_ANALYSIS']).value == 'True' + ## to-do: # - mock wrong schema path # - any other raises missed diff --git a/fre/yamltools/tests/AM5_example/COMPARE_TEST_OUTPUT_cmor.yaml b/fre/yamltools/tests/AM5_example/COMPARE_TEST_OUTPUT_cmor.yaml index 329ebb6cf..28e651911 100644 --- a/fre/yamltools/tests/AM5_example/COMPARE_TEST_OUTPUT_cmor.yaml +++ b/fre/yamltools/tests/AM5_example/COMPARE_TEST_OUTPUT_cmor.yaml @@ -1,3 +1,30 @@ +fre_properties: +- am5f7b12r1 +- am5/am5f7b12r1 +- '1979' +- '2020' +- '1980' +- '2020' +- P1Y +- 180,288 +- 720,1152 +- '0001' +- '0011' +- '0002' +- '0011' +- '0001' +- '0006' +- '0002' +- '0006' +- P5Y +- P20Y +- null +- '2000' +- /archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar +- f1a1r1 +- intel-classic +- -IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include +- -Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include grids: - gm: grid_label: gm diff --git a/fre/yamltools/tests/AM5_example/am5.yaml b/fre/yamltools/tests/AM5_example/am5.yaml index cf39a752c..c34be59f3 100644 --- a/fre/yamltools/tests/AM5_example/am5.yaml +++ b/fre/yamltools/tests/AM5_example/am5.yaml @@ -5,8 +5,8 @@ fre_properties: # amip - &EXP_AMIP_START "1979" - &EXP_AMIP_END "2020" - - &ANA_AMIP_START 1980 - - &ANA_AMIP_END 2020 + - &ANA_AMIP_START "1980" + - &ANA_AMIP_END "2020" # - &PP_AMIP_CHUNK96 "P1Y" - &PP_AMIP_CHUNK384 "P1Y" diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml b/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml index 2ed72a941..d9e244dba 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml +++ b/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml @@ -43,29 +43,6 @@ build: compileYaml: "compile.yaml" platformYaml: "wrong_platforms.yaml" -shared: - # directories shared across tools - directories: &shared_directories - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] - ptmp_dir: "/xtmp/$USER/ptmp" - fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" - - # shared pp settings - postprocess: - settings: &shared_settings - history_segment: "P1Y" - site: "ppan" - switches: &shared_switches - do_statics: True - do_timeavgs: True - clean_work: True - do_refinediag: False - do_atmos_plevel_masking: True - do_preanalysis: False - do_analysis: True - experiments: - name: "c96L65_am5f7b12r1_amip" pp: diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml b/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml index e65f3bd2b..2bbb39004 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml +++ b/fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml @@ -43,29 +43,6 @@ build: compileYaml: "wrong_compile.yaml" platformYaml: "wrong_platforms.yaml" -shared: - # directories shared across tools - directories: &shared_directories - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] - ptmp_dir: "/xtmp/$USER/ptmp" - fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" - - # shared pp settings - postprocess: - settings: &shared_settings - history_segment: "P1Y" - site: "ppan" - switches: &shared_switches - do_statics: True - do_timeavgs: True - clean_work: True - do_refinediag: False - do_atmos_plevel_masking: True - do_preanalysis: False - do_analysis: True - experiments: - name: "c96L65_am5f7b12r1_amip" pp: diff --git a/fre/yamltools/tests/AM5_example/settings.yaml b/fre/yamltools/tests/AM5_example/settings.yaml index 3e4f043d3..749dcfba5 100644 --- a/fre/yamltools/tests/AM5_example/settings.yaml +++ b/fre/yamltools/tests/AM5_example/settings.yaml @@ -22,5 +22,3 @@ postprocess: switches: clean_work: True do_atmos_plevel_masking: True - do_analysis: True - do_analysis_only: False diff --git a/fre/yamltools/tests/esm4_cmip6_ex/pp_yamls/settings.yaml b/fre/yamltools/tests/esm4_cmip6_ex/pp_yamls/settings.yaml index ef6705859..6cbcef0c3 100644 --- a/fre/yamltools/tests/esm4_cmip6_ex/pp_yamls/settings.yaml +++ b/fre/yamltools/tests/esm4_cmip6_ex/pp_yamls/settings.yaml @@ -14,5 +14,3 @@ postprocess: do_timeavgs: True clean_work: True do_atmos_plevel_masking: True - do_analysis: True - do_analysis_only: False