diff --git a/fre/__init__.py b/fre/__init__.py index 3a3702c89..bf9e46594 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -9,7 +9,7 @@ fre_logger = logging.getLogger(__name__) -FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s" +FORMAT = "[%(levelname)7s:%(filename)25s:%(funcName)24s] %(message)s" logging.basicConfig(level = logging.WARNING, format = FORMAT, filename = None, diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index b40cf228f..bf7cb16b8 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -34,9 +34,41 @@ fre_logger = logging.getLogger(__name__) -def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejobs: int = 4, +def compile_call(fremake_yaml, template_path, src_dir, bld_dir, target, platform, jobs): + """ + lkjsdlfkjs + + :param fremake_yaml: + :type fremake_yaml: + :param template_path: + :type template_path: + :param src_dir: + :type src_dir: + :param bld_dir: + :type bld_dir: + :param target: + :type target: + :param platform: + :type platform: + :param jobs: + :type jobs: + """ + fremake_build = buildBaremetal.buildBaremetal(exp=fremake_yaml["experiment"], + mkTemplatePath=template_path, + srcDir=src_dir, + bldDir=bld_dir, + target=target, + env_setup=platform["envSetup"], + jobs=jobs) + for c in fremake_yaml['src']: + fremake_build.writeBuildComponents(c) + fremake_build.writeScript() + fre_logger.info("Compile script created: %s/compile.sh", bld_dir) + +def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4, nparallel: int = 1, execute: Optional[bool] = False, - verbose: Optional[bool] = None): + verbose: Optional[bool] = False, + force_compile: Optional[bool] = False): """ This function compile_create generates the compile script for bare-metal build. @@ -55,6 +87,8 @@ def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejob :type execute: bool :param verbose: If True, increase verbosity output :type verbose: bool + :param force_compile: Re-create compile script if specified + :type force_compile: bool :raises ValueError: - Error if platform does not exist in platforms yaml configuration - Error if the mkmf template defined in platforms yaml does not exist @@ -91,6 +125,7 @@ def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejob model_yaml = yamlfre.freyaml(full_combined, fre_vars) fremake_yaml = model_yaml.getCompileYaml() + tlist = target ## Error checking the targets for target_name in tlist: target = targetfre.fretarget(target_name) @@ -117,21 +152,58 @@ def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejob model_root = platform["modelRoot"], container_flag = platform["container"]) - ## Create a list of compile scripts to run in parallel - fremake_build = buildBaremetal.buildBaremetal(exp=fremake_yaml["experiment"], - mkTemplatePath=template_path, - srcDir=src_dir, - bldDir=bld_dir, - target=target, - env_setup=platform["envSetup"], - jobs=jobs) - for c in fremake_yaml['src']: - fremake_build.writeBuildComponents(c) - fremake_build.writeScript() - fremake_build_list.append(fremake_build) - fre_logger.info("Compile script created here: %s/compile.sh", bld_dir) - + if not Path(f"{bld_dir}/compile.sh").exists(): + ## Create a list of compile scripts to run in parallel + compile_call(fremake_yaml = fremake_yaml, + template_path = template_path, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + platform = platform, + jobs = jobs) + fremake_build_list.append(f"{bld_dir}/compile.sh") + elif Path(f"{bld_dir}/compile.sh").exists() and force_compile: + # Remove old compile script + fre_logger.warning("Compile script PREVIOUSLY created: %s/compile.sh", bld_dir) + fre_logger.warning("*** REMOVING COMPILE SCRIPT ***") + Path(f"{bld_dir}/compile.sh").unlink() + + # Re-create compile script + compile_call(fremake_yaml = fremake_yaml, + template_path = template_path, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + platform = platform, + jobs = jobs) + + fremake_build_list.append(f"{bld_dir}/compile.sh") + elif Path(f"{bld_dir}/compile.sh").exists() and not force_compile: + fre_logger.warning("Compile script PREVIOUSLY created: %s/compile.sh", bld_dir) + fremake_build_list.append(f"{bld_dir}/compile.sh") + ###COMPARE THE TWO TO SEE IF IT'S CHANGED###--> filecmp or difflib + ###IF CHANGED, THROW ERROR### + ###SHOULD IT ALSO BE RE-CREATED IF CHECKOUT RE-CREATED?? --> + ### I THINK THIS WILL BE FOR "ALL" SUBTOOL### + + fre_logger.setLevel(level=logging.INFO) + fre_logger.info("Compile scripts available/generated with specified platform-target combination: ") + for i in fremake_build_list: + fre_logger.info(" - %s", i) + fre_logger.setLevel(level=logging.WARNING) + + # Returns the exit status for multiprocessing pool command if execute: if baremetal_run: - pool = Pool(processes=nparallel) # Create a multiprocessing Pool - pool.map(buildBaremetal.fremake_parallel, fremake_build_list) # process data_inputs iterable with pool + # Create a multiprocessing Pool + pool = Pool(processes=nparallel) + # process data_inputs iterable with pool + results = pool.map(buildBaremetal.fremake_parallel, fremake_build_list) + + for r in results: + for key,value in r.items(): + if key == 1: + fre_logger.error("ERROR: compile NOT successful") + fre_logger.error("Check the generated log: %s", value) + elif key == 0: + fre_logger.info("Compile successful") diff --git a/fre/make/create_makefile_script.py b/fre/make/create_makefile_script.py index 896edb079..230a3dc47 100644 --- a/fre/make/create_makefile_script.py +++ b/fre/make/create_makefile_script.py @@ -142,3 +142,4 @@ def makefile_create(yamlfile: str, platform: tuple[str], target: tuple[str]): fre_logger.setLevel(logging.INFO) fre_logger.info("Makefile created: %s/Makefile", tmp_dir) fre_logger.setLevel(former_log_level) + fre_logger.warning("") diff --git a/fre/make/fremake.py b/fre/make/fremake.py index b71e4a33b..5ba7bb1a5 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -109,19 +109,22 @@ def make_cli(): is_flag = True, default = False, help = "Force a git checkout if the source directory already exists.") +@click.option("--force-compile", + is_flag = True, + help = "Re-create the compile script in case it exists already.") @click.option("-v", "--verbose", is_flag = True, help = _VERBOSE_OPT_HELP) def all(yamlfile, platform, target, nparallel, makejobs, gitjobs, no_parallel_checkout, no_format_transfer, execute, - verbose, force_checkout): + verbose, force_checkout, force_compile): """ - Perform all fre make functions; for baremetal platforms: create checkout script, makefile, and compile scripts; for container platforms: create checkout script, makefile, Dockerfile, and createContainer script """ run_fremake_script.fremake_run( yamlfile, platform, target, nparallel, makejobs, gitjobs, no_parallel_checkout, no_format_transfer, execute, - verbose, force_checkout) + verbose, force_checkout, force_compile) @make_cli.command('checkout-script') @click.option("-y", @@ -216,14 +219,17 @@ def makefile(yamlfile, platform, target): default = False, help = """Execute the compile script immediately following its generation. The default behavior is to generate the script, but not execute.""") +@click.option("--force-compile", + is_flag = True, + help = "Re-create the compile script in case it exists already.") @click.option("-v", "--verbose", is_flag = True, help = _VERBOSE_OPT_HELP) -def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, verbose): +def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, verbose, force_compile): """ - Write the compile script """ create_compile_script.compile_create( - yamlfile, platform, target, makejobs, nparallel, execute, verbose) + yamlfile, platform, target, makejobs, nparallel, execute, verbose, force_compile) @make_cli.command('dockerfile') @click.option("-y", diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index 2a32edb7b..fd1762c8b 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -6,28 +6,67 @@ import subprocess import os +from pathlib import Path +### TODO run as a batch job on the login cluster def fremake_parallel(fremakeBuildList): """ - Brief: Called for parallel execution purposes. Runs the builds. - Param: - - fremakeBuildList : fremakeBuild object list passes by pool.map + Called for parallel execution purposes. Runs the builds. + + :param fremakeBuildList: list of compile scripts to execute + :type fremakeBuildList: ................. """ - fremakeBuildList.run() + bldDir = Path(fremakeBuildList).parent + + # Run compile script + p1 = subprocess.Popen(fremakeBuildList, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) + + # Direct output to log file as well + p2 = subprocess.Popen(["tee",f"{bldDir}/log.compile"], stdin=p1.stdout) + + # Allow process1 to receive SIGPIPE is process2 exits + p1.stdout.close() + p2.communicate() + + # wait for process1 to finish before checking return code + p1.wait() + if p1.returncode != 0: + return {1: f"{bldDir}/log.compile"} + else: + return {0: f"{bldDir}/log.compile"} class buildBaremetal(): """ - Brief: Creates the build script to compile the model - Param: - - self : The buildScript object - - exp : The experiment name - - mkTemplatePath : The template used by mkmf to compile the model - - srcDir : The source directory - - bldDir : The build directory + Class holding routines that will create the build script to compile the model. + + :ivar str exp: The experiment name + :ivar str mkTemplatePath: The template used by mkmf to compile the model + :ivar str srcDir: The source directory + :ivar str bldDir: The build directory + :ivar str target: + :ivar str env_setup: + :ivar str jobs: """ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,env_setup,jobs): """ Initialize variables and set-up the compile script. + + :param self: + :type self: + :param exp: + :type exp: + :param mkTemplatePath: + :type mkTemplatePath: + :param srcDir: + :type srcDir: + :param bldDir: + :type bldDir: + :param target: + :type target: + :param env_setup: + :type env_setup: + :param jobs: + :type jobs: """ self.e = exp self.t = target.gettargetName() @@ -57,10 +96,12 @@ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,env_setup,jobs): def writeBuildComponents(self, c): """ - Brief: Adds components to the build script - Param: - - self : The build script object - - c : Component from the compile yaml + Adds components to the build script + + :param self: The build script object + :type self: + :param c: Component from the compile yaml + :type c: """ # Shorthand for component comp = c["component"] @@ -111,9 +152,10 @@ def writeBuildComponents(self, c): ##TODO: add targets input def writeScript(self): """ - Brief: Finishes and writes the build script - Param: - - self : The buildScript object + Finishes and writes the build script + + :param self: The buildScript object + :type self: """ self.f.write(f"cd {self.bld}\n") self.f.write(f"{self.make}\n") @@ -121,30 +163,3 @@ def writeScript(self): # Make compile script executable os.chmod(self.bld+"/compile.sh", 0o744) - -## TODO run as a batch job on the login cluster - def run(self): - """ - Brief: Run the build script - Param: - - self : The dockerfile object - """ - command = [self.bld+"/compile.sh"] - - # Run compile script - p1 = subprocess.Popen(command, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - - # Direct output to log file as well - p2 = subprocess.Popen(["tee",self.bld+"/log.compile"], stdin=p1.stdout) - - # Allow process1 to receive SIGPIPE is process2 exits - p1.stdout.close() - p2.communicate() - - # wait for process1 to finish before checking return code - p1.wait() - if p1.returncode != 0: - print(f"\nThere was an error running {self.bld}/compile.sh") - print(f"Check the log file: {self.bld}/log.compile") - else: - print(f"\nSuccessful run of {self.bld}/compile.sh") diff --git a/fre/make/gfdlfremake/yamlfre.py b/fre/make/gfdlfremake/yamlfre.py index 50413c4d7..5c9654198 100644 --- a/fre/make/gfdlfremake/yamlfre.py +++ b/fre/make/gfdlfremake/yamlfre.py @@ -5,6 +5,9 @@ from jsonschema import validate, ValidationError, SchemaError from . import platformfre +import logging +fre_logger = logging.getLogger(__name__) + def parseCompile(fname,v): """ Brief: Open the yaml file and parse as fremakeYaml @@ -52,22 +55,19 @@ def __init__(self,compileinfo): try: self.yaml["src"] except: - print("You must set a src to specify the sources in modelRoot/"+self.yaml["experiment"]+"\n") - raise + raise ValueError(f"You must set a src to specify the sources in modelRoot/{self.yaml['experiment']}") ## Loop through the src array for c in self.yaml['src']: ## Check for required component name try: c['component'] except: - print("You must set the 'component' name for each src component") - raise + raise ValueError("You must set the 'component' name for each src component") ## Check for required repo url try: c['repo'] except: - print("'repo' is missing from the component "+c['component']+" in "+self.yaml["experiment"]+"\n") - raise + raise ValueError(f"'repo' is missing from the component {c['component']} in {self.yaml['experiment']}") # Check for optional branch. Otherwise set it to blank try: c['branch'] @@ -116,8 +116,7 @@ def getCompileYaml(self): try: self.yaml except: - print ("You must initialize the compile YAML object before you try to get the yaml \n") - raise + raise ValueError("You must initialize the compile YAML object before you try to get the yaml") return self.yaml class freyaml(): @@ -163,7 +162,7 @@ def __init__(self,combinedyaml,v): schema = json.loads(s) validate(instance=self.freyaml,schema=schema) - print("\nCOMBINED YAML VALID") + fre_logger.info(" *** COMBINED YAML VALID ***") def getCompileYaml(self): """ diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index ec506cbc2..40d2c841b 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -22,7 +22,8 @@ def fremake_run(yamlfile:str, platform:str, target:str, no_format_transfer: Optional[bool] = False, execute: Optional[bool] = False, verbose: Optional[bool] = None, - force_checkout: Optional[bool] = False): + force_checkout: Optional[bool] = False, + force_compile: Optional[bool] = False): """ Runs all of fre make code @@ -49,13 +50,15 @@ def fremake_run(yamlfile:str, platform:str, target:str, :type execute: bool :param force_checkout: Re-create the checkout script if changes were made to configurations :type force_checkout: bool + :param force_compile: Re-create compile script if specified + :type force_compile: bool :param verbose: Increase verbosity output :type verbose: bool """ -# if verbose: -# fre_logger.setLevel(level = logging.DEBUG) -# else: -# fre_logger.setLevel(level = logging.INFO) + if verbose: + fre_logger.setLevel(level = logging.DEBUG) + else: + fre_logger.setLevel(level = logging.INFO) # Define variables name = yamlfile.split(".")[0] @@ -102,7 +105,7 @@ def fremake_run(yamlfile:str, platform:str, target:str, #compile fre_logger.info("Running fre make: calling compile_create") compile_create(yamlfile, bm_platforms, target, makejobs, nparallel, - execute, verbose) + execute, verbose, force_compile) if container_platforms: fre_logger.info("Running fre make: calling dockerfile_create") dockerfile_create(yamlfile, container_platforms, target, execute, no_format_transfer) diff --git a/fre/make/tests/compilation/test_run_fremake_builds.py b/fre/make/tests/compilation/test_run_fremake_builds.py index 639107dfe..126e56a37 100644 --- a/fre/make/tests/compilation/test_run_fremake_builds.py +++ b/fre/make/tests/compilation/test_run_fremake_builds.py @@ -60,7 +60,8 @@ def test_run_fremake_serial_compile(): os.environ["TEST_BUILD_DIR"] = SERIAL_TEST_PATH run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, nparallel=1, makejobs=1, gitjobs=1, no_parallel_checkout=False, - no_format_transfer=True, execute=True, verbose=VERBOSE) + no_format_transfer=True, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path( f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() @@ -71,7 +72,8 @@ def test_run_fremake_multijob_compile(): os.environ["TEST_BUILD_DIR"] = MULTIJOB_TEST_PATH run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, nparallel=1, makejobs=4, gitjobs=4, no_parallel_checkout=False, - no_format_transfer=False, execute=True, verbose=VERBOSE) + no_format_transfer=False, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path( f"{MULTIJOB_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() @@ -81,7 +83,8 @@ def test_run_fremake_container_build(): ''' checks image creation for the container build''' run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, nparallel=1, makejobs=1, gitjobs=1, no_parallel_checkout=True, - no_format_transfer=False, execute=True, verbose=VERBOSE) + no_format_transfer=False, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path("null_model_full-debug.sif").exists() @pytest.mark.skipif(not can_container, reason="missing podman/apptainer") @@ -90,7 +93,8 @@ def test_run_fremake_container_build_specified_out(): os.environ["TEST_BUILD_DIR"] = CONTAINER_BUILD_TEST_PATH run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM_2, TARGET, nparallel=1, makejobs=1, gitjobs=1, no_parallel_checkout=True, - no_format_transfer=False, execute=True, verbose=VERBOSE) + no_format_transfer=False, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path( f"{CONTAINER_BUILD_TEST_PATH}/fremake_canopy/test/null_model_full-debug.sif").exists() @@ -101,7 +105,8 @@ def test_run_fremake_container_build_notransfer(): os.remove("createContainer.sh") run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, nparallel=1, makejobs=1, gitjobs=1, no_parallel_checkout=True, - no_format_transfer=True, execute=True, verbose=VERBOSE) + no_format_transfer=True, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) def test_run_fremake_cleanup(): ''' removes directories created by the test and checks to make sure they're gone ''' @@ -124,7 +129,8 @@ def test_run_fremake_container_build_fail(): # Create the createContainer.sh script but do not run run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, nparallel=1, makejobs=1, gitjobs=1, no_parallel_checkout=True, - no_format_transfer=False, execute=False, verbose=VERBOSE) + no_format_transfer=False, execute=False, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path(f"{currPath}/createContainer.sh").exists() # Alter script to fail diff --git a/fre/make/tests/test_create_checkout.py b/fre/make/tests/test_create_checkout.py index f1a0c3110..3c1fe63a6 100644 --- a/fre/make/tests/test_create_checkout.py +++ b/fre/make/tests/test_create_checkout.py @@ -140,11 +140,12 @@ def test_bm_checkout_force_checkout(caplog, monkeypatch): execute = False, force_checkout = True) + checkout_script = f"{OUT}/fremake_canopy/test/null_model_full/src/checkout.sh" # Check it exists, check output, check content - assert all([Path(f"{OUT}/fremake_canopy/test/null_model_full/src/checkout.sh").exists(), + assert all([Path(checkout_script).exists(), "Checkout script PREVIOUSLY created" in caplog.text, "*** REMOVING CHECKOUT SCRIPT ***" in caplog.text, - "Checkout script created" in caplog.text]) + f"Checkout script created: {checkout_script}" in caplog.text]) # Check one expected line is now populating the re-created checkout script expected_line = f"({EXPECTED_LINE}) &" @@ -194,7 +195,7 @@ def test_container_checkout_force_checkout(caplog): assert all([Path(f"{mock_checkout}/checkout.sh").exists(), "Checkout script PREVIOUSLY created" in caplog.text, "*** REMOVING CHECKOUT SCRIPT ***" in caplog.text, - "Checkout script created in ./tmp" in caplog.text]) + f"Checkout script created: ./tmp/{CONTAINER_PLATFORM[0]}/checkout.sh" in caplog.text]) # Check for an expected line that should be populating the re-created checkout script # Check no parenthesis (no parallel checkouts) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index 285a2be5d..a906baccd 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -1,7 +1,6 @@ """ Test fre make compile-script """ -import os import shutil from pathlib import Path @@ -35,6 +34,13 @@ else: Path(OUT).mkdir(parents=True,exist_ok=True) +@pytest.fixture(name="stdout_fixture", autouse=True) +def test_out_fixture(monkeypatch): + """ + """ + # Set environment variable for use in ci.gnu platform + monkeypatch.setenv("TEST_BUILD_DIR", OUT) + def test_modelyaml_exists(): """ Check the model yaml exists @@ -53,13 +59,10 @@ def test_platformyaml_exists(): """ assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() -def test_compile_creation(): +def test_compile_creation(stdout_fixture): """ Check for the creation of the compile script """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - plat = PLATFORM[0] targ = TARGET[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" @@ -71,19 +74,47 @@ def test_compile_creation(): makejobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) # Check for creation of compile script assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() -def test_compile_executable_failure(): +def test_force_compile_creation(caplog, stdout_fixture): + """ + """ + plat = PLATFORM[0] + targ = TARGET[0] + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Check compile script still exists from last test + assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() + + # 2nd compile script creation + create_compile_script.compile_create(yamlfile = yamlfile_path, + platform = PLATFORM, + target = TARGET, + njobs = 4, + nparallel = 1, + execute = False, + verbose = False, + force_compile = True) + + + # Check for re-creation of compile script + print(caplog.text) + compile_script = f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh" + assert Path(compile_script).exists() + assert f"Compile script PREVIOUSLY created: {compile_script}" in caplog.text + assert "*** REMOVING COMPILE SCRIPT ***" in caplog.text + assert "Compile scripts available/generated with specified platform-target combination" in caplog.text + assert f" - {compile_script}" in caplog.text + +def test_compile_executable_failure(stdout_fixture): """ Check for the failure in execution of the compile script. Fails because it would need the makefile and checked out source code. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - plat = PLATFORM[0] targ = TARGET[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" @@ -95,7 +126,8 @@ def test_compile_executable_failure(): makejobs = 4, nparallel = 1, execute = True, - verbose = False) + verbose = False, + force_compile = False) # Check for creation of compile script, FMS directory, # log.compile file, the executable @@ -105,14 +137,11 @@ def test_compile_executable_failure(): assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/null_model_full.x").exists() == False @pytest.mark.xfail(raises=ValueError) -def test_bad_platform(): +def test_bad_platform(stdout_fixture): """ Check for the failure of compile script creation due to a bad platform passed. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Create the compile script @@ -122,16 +151,14 @@ def test_bad_platform(): makejobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) -def test_bad_platform_compilelog(): +def test_bad_platform_compilelog(stdout_fixture): """ Check that compile log still created from the failure of compile script creation due to a bad platform passed. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" try: @@ -142,19 +169,17 @@ def test_bad_platform_compilelog(): makejobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) except: assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{BAD_PLATFORM}-{TARGET}/exec/log.compile") @pytest.mark.xfail(raises=ValueError) -def test_bad_target(): +def test_bad_target(stdout_fixture): """ Check for the failure of compile script creation due to a bad target passed. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Create the compile script @@ -164,16 +189,14 @@ def test_bad_target(): makejobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) -def test_bad_target_compilelog(): +def test_bad_target_compilelog(stdout_fixture): """ Check that compile log still created from the failure of compile script creation due to a bad target passed. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" try: @@ -184,17 +207,15 @@ def test_bad_target_compilelog(): makejobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) except: assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{BAD_PLATFORM}-{TARGET}/exec/log.compile") -def test_multi_target(): +def test_multi_target(stdout_fixture): """ Check for the creation of the compile script for each target passed """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Create the compile script @@ -204,7 +225,8 @@ def test_multi_target(): makejobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI_TARGET[0]}/exec/compile.sh").exists() assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI_TARGET[1]}/exec/compile.sh").exists() diff --git a/fre/make/tests/test_run_fremake.py b/fre/make/tests/test_run_fremake.py index 017fcc9b8..623ed4c57 100644 --- a/fre/make/tests/test_run_fremake.py +++ b/fre/make/tests/test_run_fremake.py @@ -57,7 +57,7 @@ def test_bad_platform_option(): run_fremake_script.fremake_run(YAMLPATH, BADOPT, TARGET, nparallel=False, makejobs=1, gitjobs=1, no_parallel_checkout=False, no_format_transfer=False, execute=False, verbose=VERBOSE, - force_checkout=False) + force_checkout=False, force_compile=False) @pytest.mark.xfail() def test_bad_target_option(): @@ -65,7 +65,7 @@ def test_bad_target_option(): run_fremake_script.fremake_run(YAMLPATH, PLATFORM, BADOPT, nparallel=False, makejobs=1, gitjobs=1, no_parallel_checkout=False, no_format_transfer=False, execute=False, verbose=VERBOSE, - force_checkout=False) + force_checkout=False, force_compile=False) @pytest.mark.xfail() def test_bad_yamlpath_option(): @@ -73,7 +73,7 @@ def test_bad_yamlpath_option(): run_fremake_script.fremake_run(BADOPT[0], PLATFORM, TARGET, nparallel=False, makejobs=1, gitjobs=1, no_parallel_checkout=False, no_format_transfer=False, execute=False, verbose=VERBOSE, - force_checkout=False) + force_checkout=False, force_compile=False) # tests script/makefile creation without executing (serial compile) # first test runs the run-fremake command, subsequent tests check for creation of scripts @@ -83,7 +83,7 @@ def test_run_fremake_serial(): run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, nparallel=False, makejobs=1, gitjobs=1, no_parallel_checkout=False, no_format_transfer=False, execute=False, verbose=VERBOSE, - force_checkout=False) + force_checkout=False, force_compile=False) def test_run_fremake_compile_script_creation_serial(): ''' check for compile script creation from previous test ''' @@ -128,7 +128,7 @@ def test_run_fremake_multijob(): run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, nparallel=True, makejobs=4, gitjobs=4, no_parallel_checkout=True, no_format_transfer=False, execute=False, verbose=VERBOSE, - force_checkout=False) + force_checkout=False, force_compile=False) def test_run_fremake_compile_script_creation_multijob(): ''' check for compile script creation from previous test ''' @@ -151,7 +151,7 @@ def test_run_fremake_container(): run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, nparallel=False, makejobs=1, gitjobs=1, no_parallel_checkout=True, no_format_transfer=False, execute=False, verbose=VERBOSE, - force_checkout=False) + force_checkout=False, force_compile=False) def test_run_fremake_build_script_creation_container(): ''' checks container build script creation from previous test ''' @@ -179,7 +179,7 @@ def test_run_fremake_container_2stage(): run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLAT2, TARGET, nparallel=False, makejobs=1, gitjobs=1, no_parallel_checkout=True, no_format_transfer=False, execute=False, verbose=VERBOSE, - force_checkout=False) + force_checkout=False, force_compile=False) def test_run_fremake_build_script_creation_container_2stage(): ''' checks container build script creation from previous test ''' diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index bf12c33ff..35bfa6303 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -70,8 +70,8 @@ def test_cli_fre_cmor_help_and_debuglog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long - log_text_line_2='[DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_2='[ DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] @@ -89,7 +89,7 @@ def test_cli_fre_cmor_help_and_infolog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0]