From 25baa70ddcfd3010f0edc5ed2d2f3a9a43531176 Mon Sep 17 00:00:00 2001 From: james <81617086+je-cook@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:21:35 +0100 Subject: [PATCH 01/15] tracking --- tracking/git.py | 41 ++++++---- tracking/run_tracking_inputs.py | 3 +- tracking/tracking_data.py | 134 +++++++++++++++++++------------- 3 files changed, 111 insertions(+), 67 deletions(-) diff --git a/tracking/git.py b/tracking/git.py index 558c18e220..0d8fa2dc89 100644 --- a/tracking/git.py +++ b/tracking/git.py @@ -1,19 +1,26 @@ -"""Simple submodule to provide access to some git attributes about the current repository""" +"""Simple submodule to provide access to some git attributes about the repository""" import subprocess # noqa: S404 from pathlib import Path -def git_commit_message(directory=None) -> str: - """Get the commit message for `directory` or, if `directory` is not provided, the current working directory. - - Will raise a subprocess.CalledProcessError if the directory checked is not a git repository. +def git_commit_message(directory=str | bytes | None) -> str: + """Get the commit message for `directory` or the current directory if not provided + Parameters + ---------- :param directory: the directory to get the commit message at - :type directory: str | bytes | path-like - :return commit_message: the commit message for the git repository - :type commit_message: str + Returns + ------- + : + the commit message for the git repository + + Raises + ------ + subprocess.CalledProcessError + if the directory checked is not a git repository. + """ if directory is None: directory = Path.cwd() @@ -30,15 +37,21 @@ def git_commit_message(directory=None) -> str: def git_commit_hash(directory=None) -> str: - """Get the commit hash for `directory` or, if `directory` is not provided, the current working directory. + """Get the commit hash for `directory` or the current directory if not provided - Will raise a subprocess.CalledProcessError if the directory checked is not a git repository. + Parameters + ---------- + :param directory: the directory to get the commit hash at - :param directory: the directory to get the commit hash for - :type directory: str | bytes | path-like + Returns + ------- + : + the commit hash for the git repository - :return commit_hash: the commit hash for the git repository - :type commit_hash: str + Raises + ------ + subprocess.CalledProcessError + if the directory checked is not a git repository. """ if directory is None: directory = Path.cwd() diff --git a/tracking/run_tracking_inputs.py b/tracking/run_tracking_inputs.py index 1bad54f125..7960623468 100755 --- a/tracking/run_tracking_inputs.py +++ b/tracking/run_tracking_inputs.py @@ -75,7 +75,8 @@ def tracking(arguments): type=Path, default=None, help="A JSON file containing a list of variables to track." - "See the description of DEFAULT_TRACKING_VARIABLES for details on formatting the strings in the list.", + "See the description of DEFAULT_TRACKING_VARIABLES " + "for details on formatting the strings in the list.", ) subparser_trk.add_argument("--strict", action="store_true", default=False) diff --git a/tracking/tracking_data.py b/tracking/tracking_data.py index 68608fbd5c..e9af8d9230 100644 --- a/tracking/tracking_data.py +++ b/tracking/tracking_data.py @@ -1,33 +1,48 @@ """The tracking tool aims to provide a CLI for two important tracking tools. -We can generate tracking data for a given run of PROCESS, which corresponds to one output MFile. -When generating tracking data, we store metadata about the run, such as time and commit message, -and also variable data: the value of a given variable during this run; this is all stored in one -JSON file. - -* Run title: the name of the run input, and hence output, file (e.g. baseline 2018). -* Tracking history: the data held within all JSON files in the database that span many variables, -many different run titles and spans a period of time. -* Variable history: the data held within all JSON files in the database for a given variable; -not all run titles will have a history for each variable being tracked -* A run of PROCESS: refers to the output MFile from the running of one input file -* JSON file: holds the metadata, and tracking data, generated by a run of PROCESS - -We can also plot a database of JSON files. The tracking history (ie all the data) must be loaded -before being processed into a dataframe. This dataframe is then processed into a tracking dashboard: - -* The dashboard: represented by the entire html file generated, contains many panels -* Panels: each parent module (or variable namespace) has its own tab which contains many graphs -* Graphs: each graph shows the variable history of one variable, it will have many lines -* Lines: each line represents many datapoints for a single variable, over a period of time, -that all come from the same run title -* Datapoints: the value of a variable at a point in time during a given run +We can generate tracking data for a given run of PROCESS, +which corresponds to one output MFile. +When generating tracking data, we store metadata about the run, +such as time and commit message, +and also variable data: the value of a given variable during this run; +this is all stored in one JSON file. + +* Run title: + the name of the run input, and hence output, file (e.g. baseline 2018). +* Tracking history: + the data held within all JSON files in the database that span many variables, + many different run titles and spans a period of time. +* Variable history: + the data held within all JSON files in the database for a given variable; + not all run titles will have a history for each variable being tracked +* A run of PROCESS: + refers to the output MFile from the running of one input file +* JSON file: + holds the metadata, and tracking data, generated by a run of PROCESS + +We can also plot a database of JSON files. +The tracking history (ie all the data) must be loaded +before being processed into a dataframe. +This dataframe is then processed into a tracking dashboard: + +* The dashboard: + represented by the entire html file generated, contains many panels +* Panels: + each parent module (or variable namespace) has its own tab which contains many graphs +* Graphs: + each graph shows the variable history of one variable, it will have many lines +* Lines: + each line represents many datapoints for a single variable, + over a period of time, that all come from the same run title +* Datapoints: + the value of a variable at a point in time during a given run To add a variable to track: Add the variable to ProcessTracker.tracking_variables (in this file). -If the variable is not a fortran module variable, ensure to override its parent module name +If the variable is not a fortran module variable, +ensure to override its parent module name e.g. FOO.bar says `bar`'s parent module is `FOO`. """ @@ -141,7 +156,9 @@ class TrackingFile: - """Acts as the data storage for a given JSON file ie. holds the data from a run of PROCESS + """Acts as the data storage for a given JSON file + + ie. holds the data from a run of PROCESS e.g. starfire_MFILE--