diff --git a/changelog-entries/811.md b/changelog-entries/811.md new file mode 100644 index 000000000..8a8bfc6ae --- /dev/null +++ b/changelog-entries/811.md @@ -0,0 +1 @@ +- Store each `reference-results-metadata.txt` inside the case directory of its reference results archive instead of sharing one metadata file between all archives of a tutorial. The SHA256 checksum of the outer archive is no longer included, because the metadata now lives inside that archive [#882](https://github.com/precice/tutorials/pull/882). diff --git a/tools/tests/README.md b/tools/tests/README.md index 3321ed7d5..f0e44adb8 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -107,6 +107,7 @@ To reproduce the comparison locally, use the [same fieldcompare command](https:/ ```bash fieldcompare dir precice-exports/ reference-results-unpacked// \ --ignore-missing-reference-files \ + --exclude-files reference-results-metadata.txt \ --ignore-unsupported-file-formats \ -rtol 3e-7 ``` @@ -176,6 +177,10 @@ The two options cannot be combined: defining any overrides to `reference_version The results will be added to a Git LFS, but you will need special push access: just use the aforementioned GitHub Actions workflow, instead. +Each generated reference archive contains a `reference-results-metadata.txt` file +inside its case directory. This file records the component versions and machine +information used to generate that archive. + #### External test sources Local test cases are defined in the `tutorials:` list. For test cases maintained in other git repositories or available as archives, use the `external:` list. A test suite may define both lists so that external cases can be referenced via YAML anchors alongside local tutorials (for example in the `release` suite). @@ -284,7 +289,7 @@ Metadata and workflow/script files: - `docker-compose.template.yaml`: Describes how to prepare each test (Docker Compose service template) - `docker-compose.field_compare.template.yaml`: Describes how to compare results with fieldcompare (Docker Compose service template) - `components.yaml`: Declares the available components and their parameters/options - - `reference_results.metadata.template`: Template for reporting the versions used to generate the reference results + - `reference-results-metadata.txt.template`: Template for reporting the versions and machine used to generate each reference results archive - `reference_versions.yaml`: List of arguments to use for generating the reference results - `tests.yaml`: Declares the available tests, grouped in test suites diff --git a/tools/tests/docker-compose.field_compare.template.yaml b/tools/tests/docker-compose.field_compare.template.yaml index 65590ab61..8a789b109 100644 --- a/tools/tests/docker-compose.field_compare.template.yaml +++ b/tools/tests/docker-compose.field_compare.template.yaml @@ -6,4 +6,4 @@ services: command: - /runs/{{ tutorial_folder }}/{{ precice_output_folder }} - /runs/{{ tutorial_folder }}/{{ reference_output_folder }} - - "-rtol {{ tolerance }} --ignore-missing-reference-files --diff" + - "-rtol {{ tolerance }} --ignore-missing-reference-files --exclude-files reference-results-metadata.txt --ignore-unsupported-file-formats --diff" diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 0787cc6e4..c2f3b4425 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -1,5 +1,5 @@ import argparse -from metadata_parser.metdata import Tutorials, ReferenceResult +from metadata_parser.metdata import Tutorials from systemtests.TestSuite import TestSuites from systemtests.SystemtestArguments import SystemtestArguments from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, ITERATIONS_LOGS_DIR @@ -7,7 +7,6 @@ from typing import List import shutil from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR -import hashlib from jinja2 import Environment, FileSystemLoader import tarfile import subprocess @@ -23,8 +22,9 @@ def create_reference_tar_gz( exports_dir: Path, output_filename: Path, iterations_logs: List[tuple[str, Path]], + metadata: str, ) -> None: - """Archive precice-exports and optional iterations logs as separate top-level tar members.""" + """Archive precice-exports, metadata, and optional iterations logs.""" stem = output_filename.name.replace(".tar.gz", "") exports_staging = system_test_dir / f".{stem}_reference_exports_staging" logs_staging = system_test_dir / f".{stem}_reference_logs_staging" @@ -33,6 +33,7 @@ def create_reference_tar_gz( shutil.rmtree(staging) shutil.copytree(exports_dir, exports_staging) try: + (exports_staging / "reference-results-metadata.txt").write_text(metadata) with tarfile.open(output_filename, "w:gz") as tar: tar.add(exports_staging, arcname=stem) if iterations_logs: @@ -75,36 +76,20 @@ def command_is_avail(command: str): def render_reference_results_info( - reference_results: List[ReferenceResult], + archive_name: str, arguments_used: SystemtestArguments, time: str): - def sha256sum(filename): - # Implementation from https://stackoverflow.com/a/44873382/2254346, - # compatible with Python 3.10. - h = hashlib.sha256() - mv = memoryview(bytearray(128 * 1024)) - with open(filename, 'rb', buffering=0) as f: - while n := f.readinto(mv): - h.update(mv[:n]) - return h.hexdigest() - - files = [] - for reference_result in reference_results: - files.append({ - 'sha256': sha256sum(reference_result.path), - 'time': time, - 'name': reference_result.path.name, - }) uname, lscpu = get_machine_informations() render_dict = { 'arguments': arguments_used.arguments, - 'files': files, + 'archive_name': archive_name, + 'time': time, 'uname': uname, 'lscpu': lscpu, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) - template = jinja_env.get_template("reference_results.metadata.template") + template = jinja_env.get_template("reference-results-metadata.txt.template") return template.render(render_dict) @@ -197,7 +182,6 @@ def main(): max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, run_before=run_before, run_after=run_after)) - reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') logging.info(f"About to run the following tests {systemtests_to_run}") @@ -209,21 +193,25 @@ def main(): logging.info(f"Running {systemtest} took {elapsed_time:^.1f} seconds") if not result.success: raise RuntimeError(f"Failed to execute {systemtest}") - reference_result_per_tutorial[systemtest.tutorial] = [] # Put the tar.gz in there for systemtest in systemtests_to_run: reference_result_folder = systemtest.get_system_test_dir() / PRECICE_REL_OUTPUT_DIR - reference_result_per_tutorial[systemtest.tutorial].append(systemtest.reference_result) # create folder if needed systemtest.reference_result.path.parent.mkdir(parents=True, exist_ok=True) if reference_result_folder.exists(): collected = systemtest._collect_iterations_logs(systemtest.get_system_test_dir()) + metadata = render_reference_results_info( + systemtest.reference_result.path.name, + build_args, + current_time_string, + ) create_reference_tar_gz( systemtest.get_system_test_dir(), reference_result_folder, systemtest.reference_result.path, collected, + metadata, ) if collected: logging.info( @@ -235,15 +223,6 @@ def main(): raise RuntimeError( f"Error executing: \n {systemtest} \n Could not find result folder {reference_result_folder}\n Probably the tutorial did not run through properly. Please check corresponding logs") - # write readme - for tutorial in reference_result_per_tutorial.keys(): - reference_results_dir = tutorial.path / "reference-results" - reference_results_dir.mkdir(parents=True, exist_ok=True) - with open(reference_results_dir / "reference_results.metadata", 'w') as file: - ref_results_info = render_reference_results_info( - reference_result_per_tutorial[tutorial], build_args, current_time_string) - logging.info(f"Writing results for {tutorial.name}") - file.write(ref_results_info) logging.info(f"Done. Please make sure to manually have a look into the reference results before making a PR.") diff --git a/tools/tests/reference-results-metadata.txt.template b/tools/tests/reference-results-metadata.txt.template new file mode 100644 index 000000000..2e196c865 --- /dev/null +++ b/tools/tests/reference-results-metadata.txt.template @@ -0,0 +1,33 @@ + + +# Reference Results + +This file describes the reference results in `{{ archive_name }}` and includes +the arguments and machine used to generate them. + +## Archive + +| name | generated at | +|------|--------------| +| {{ archive_name }} | {{ time }} | + +## Arguments used to generate the results + +| name | value | +|------|------| +{% for name,value in arguments.items() -%} +| {{ name }} | {{ value }} | +{% endfor -%} + + +## Information about the machine + +### uname -a + +{{ uname }} + +### lscpu + +{{ lscpu }} \ No newline at end of file diff --git a/tools/tests/reference_results.metadata.template b/tools/tests/reference_results.metadata.template deleted file mode 100644 index d7f189fe2..000000000 --- a/tools/tests/reference_results.metadata.template +++ /dev/null @@ -1,34 +0,0 @@ - - -# Reference Results - -This file contains an overview of the results over the reference results as well as the arguments used to generate them. -We also include some information on the machine used to generate them - -## List of files - -| name | time | sha256 | -|------|------|-------| -{% for file in files -%} -| {{ file.name }} | {{ file.time }} | {{ file.sha256 }} | -{% endfor %} -## List of arguments used to generate the files - -| name | value | -|------|------| -{% for name,value in arguments.items() -%} -| {{ name }} | {{ value }} | -{% endfor -%} - - -## Information about the machine - -### uname -a - -{{ uname }} - -### lscpu - -{{ lscpu }} \ No newline at end of file