[benchmarker] Add benchmarker configuration and report definitions

monitoring/benchmarker/configurations/actions.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+from implicitdict import ImplicitDict
+
+
+class BenchmarkActionName(str):
+    """Unique (within benchmark configuration) name for an action to perform, for instance at setup and/or teardown of one or more scenarios."""
+
+    pass
+
+
+class RunCommandActionSpecification(ImplicitDict):
+    """Shell command to run as a benchmark action."""
+
+    command: str
+    """Shell command to run."""
+
+    path: str
+    """Working folder in which to run the command.  `$REPO_ROOT` will be replaced with the root folder of the repo."""
+
+    env: dict[str, str]
+    """Override each environment variable key with the specified value before running the command."""
+
+
+class BenchmarkActionSpecification(ImplicitDict):
+    name: BenchmarkActionName
+
+    run_command: Optional[RunCommandActionSpecification]

monitoring/benchmarker/configurations/artifacts/artifact.py

@@ -0,0 +1,15 @@
+from typing import Optional
+
+from implicitdict import ImplicitDict
+
+from monitoring.benchmarker.configurations.artifacts.matplotlib_figure import (
+    MatplotlibFigureSpecification,
+)
+from monitoring.benchmarker.configurations.artifacts.raw_report import (
+    RawReportSpecification,
+)
+
+
+class ArtifactSpecification(ImplicitDict):
+    raw_report: Optional[RawReportSpecification]
+    matplotlib_figure: Optional[MatplotlibFigureSpecification]

monitoring/benchmarker/configurations/artifacts/matplotlib_figure.py

@@ -0,0 +1,73 @@
+from enum import StrEnum
+from typing import Optional
+
+from implicitdict import ImplicitDict
+
+from monitoring.monitorlib.expressions.types import ASTExpression, SymbolExpression
+
+
+class AxisSpecification(ImplicitDict):
+    label: Optional[str]
+
+
+class XYPlotType(StrEnum):
+    Scatter = "Scatter"
+
+
+class XYPlotSpecification(ImplicitDict):
+    """Specification for a plot (artist) to show on a matplotlib Axes.
+
+    When evaluating expressions, the BenchmarkRunReport will be available as the `report` symbol."""
+
+    type: XYPlotType
+
+    evaluation_context: Optional[list[SymbolExpression]]
+    """Symbols available to other expressions in this plot specification."""
+
+    x_data_expr: Optional[ASTExpression]
+    """List of X data values for XY points.
+
+    Must have the same number of entries as y_data.
+    Defaults to 1, 2, 3, ..., N for N y_data values."""
+
+    y_data_expr: ASTExpression
+    """List of Y data values for XY points."""
+
+    render_expr: Optional[ASTExpression]
+    """If specified, whether this plot should be rendered (boolean).  Default true."""
+
+
+class SubplotSpecification(ImplicitDict):
+    title: Optional[str]
+
+    evaluation_context: Optional[list[SymbolExpression]]
+    """Symbols available to other expressions in this subplot specification."""
+
+    x_axis: Optional[AxisSpecification]
+    y_axis: Optional[AxisSpecification]
+    xy_plots: list[XYPlotSpecification]
+
+
+class SubfigureSpecification(ImplicitDict):
+    title: Optional[str]
+
+    n_subplot_rows: int = 1
+    n_subplot_cols: int = 1
+
+    evaluation_context: Optional[list[SymbolExpression]]
+    """Symbols available to other expressions in this subfigure specification."""
+
+    subplots: list[SubplotSpecification]
+
+
+class MatplotlibFigureSpecification(ImplicitDict):
+    name: str
+    """Machine-level name for this figure.  Used as the output file name."""
+
+    n_subfigure_rows: int = 1
+    n_subfigure_cols: int = 1
+
+    evaluation_context: Optional[list[SymbolExpression]]
+    """Symbols available to other expressions in this figure specification."""
+
+    subfigures: list[SubfigureSpecification]

monitoring/benchmarker/configurations/artifacts/raw_report.py

@@ -0,0 +1,6 @@
+from implicitdict import ImplicitDict
+
+
+class RawReportSpecification(ImplicitDict):
+    name: str
+    """Machine-level name for this report.  Used as the output file name."""

monitoring/benchmarker/configurations/configuration.py

@@ -0,0 +1,36 @@
+from typing import Optional
+
+from implicitdict import ImplicitDict
+
+from monitoring.benchmarker.configurations.actions import (
+    BenchmarkActionSpecification,
+)
+from monitoring.benchmarker.configurations.artifacts.artifact import (
+    ArtifactSpecification,
+)
+from monitoring.benchmarker.configurations.loads import BenchmarkLoadSpecification
+from monitoring.benchmarker.configurations.scenarios import (
+    BenchmarkScenarioSpecification,
+)
+from monitoring.benchmarker.configurations.users import BenchmarkUserSpecification
+from monitoring.uss_qualifier.resources.definitions import ResourceCollection
+
+
+class BenchmarkConfiguration(ImplicitDict):
+    resources: Optional[ResourceCollection]
+    """Pool of uss_qualifier resources available for use by other resources in this benchmark configuration."""
+
+    actions: Optional[list[BenchmarkActionSpecification]]
+    """Actions available to be performed during the benchmarker run."""
+
+    user_types: list[BenchmarkUserSpecification]
+    """Types of users available to load the system under test during the benchmarker run."""
+
+    loads: list[BenchmarkLoadSpecification]
+    """Loads that can be applied to the system under test during the benchmarker run."""
+
+    scenarios: list[BenchmarkScenarioSpecification]
+    """The sequential list of scenarios to perform in the benchmarker run."""
+
+    artifacts: Optional[list[ArtifactSpecification]]
+    """Artifacts to produce from the data collected during the benchmarker run."""