diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e80c736c0..0a34a268b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,8 @@ jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} permissions: contents: read diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 178116ba6..b86761eeb 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -4,6 +4,9 @@ name: Merge-Gate on: workflow_call: + secrets: + SONAR_TOKEN: + required: true jobs: run-fast-checks: @@ -47,14 +50,12 @@ jobs: needs: - approve-run-slow-tests uses: ./.github/workflows/slow-checks.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. permissions: contents: read merge-gate-extension: name: Extension uses: ./.github/workflows/merge-gate-extension.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. permissions: contents: read diff --git a/.github/workflows/periodic-validation.yml b/.github/workflows/periodic-validation.yml index 01b25f5b3..62834750f 100644 --- a/.github/workflows/periodic-validation.yml +++ b/.github/workflows/periodic-validation.yml @@ -46,7 +46,6 @@ jobs: uses: ./.github/workflows/slow-checks.yml needs: - restrict-to-default-branch - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. permissions: contents: read diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index f10f6f857..c81d31e44 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,8 +2,36 @@ ## Summary -Updated the nox DB-version default to come from `BaseConfig` instead of the hardcoded `7.1.9`, -so ITDE-related test flows use the configured Exasol baseline and unit-test help no longer advertises `--db-version`. +In this major release, several modifications were made to the PTB's workflow templates and actions: + +* the default DB-version was updated to come from `BaseConfig` instead of the + hardcoded `7.1.9`, so ITDE-related test flows use the configured Exasol baseline + and unit-test help no longer advertises `--db-version`. +* the `github_template_dict.custom_workflows` entry now auto-detects secret names + from custom workflow files and passes them into PTB-controlled workflow templates. + For example: + + ```yaml + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + SONAR_TOKEN: + required: true + ``` +* the Python environment GitHub action now accepts `extras` as a comma-separated + list, which makes it easier to pass multiple optional dependency groups in one + value. Additionally, it supports `all-extras`, so that all extras are installed + without further specification needed. +* the new `workflow:audit` Nox session runs `zizmor` against GitHub Actions and + reusable workflows, so security checks are part of the normal `checks.yml` + pipeline instead of being a separate manual step. It also keeps the audit + configuration in the project root via `.zizmor.yml`; see the + [zizmor configuration guide](https://exasol.github.io/python-toolbox/main/user_guide/features/managing_dependencies/zizmor_configuration.html) + and the + [troubleshooting guide for findings](https://exasol.github.io/python-toolbox/main/user_guide/troubleshooting/handle_zizmor_findings.html) + for details on tuning or suppressing findings locally. ## Feature @@ -18,10 +46,12 @@ so ITDE-related test flows use the configured Exasol baseline and unit-test help ## Feature * #878: Added Nox session `workflow:audit` which uses `zizmor` and added it in `checks.yml` +* #872: Added `custom_workflows` to `github_template_dict` for automatic custom workflow secret extraction ## Refactoring * #744: Extracted shared minimum-version selection logic into `minimum_declared_version()` +* #699: Switched `extras` in the Python environment GitHub action to comma-separation ## Documentation diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index 35db31df4..9b3e85b9b 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -36,10 +36,15 @@ compare the rendered workflow templates against the files in ``.github/workflows Workflows --------- -The PTB has three categories of workflows: +The PTB allows for two categories of workflows: #. those maintained by the PTB, which can be modified using the :ref:`workflow_patcher`. - #. those which are seeded by the PTB but owned and maintained by the project after initial creation. - #. those which extend the PTB-provided workflows and are maintained by the project (not the PTB). + #. custom workflows, which are project-owned. + +Custom workflows can optionally be +* seeded by the PTB, i.e. PTB generates an initial version but ignores future changes. +* extend PTB-provided workflows, i.e. ending in `-extension.yml` + +Besides that, you can also create individual workflow files which are ignored by the PTB. Maintained by the PTB ^^^^^^^^^^^^^^^^^^^^^ @@ -108,8 +113,26 @@ Maintained by the PTB the action, and uploads the results to Sonar. -Not Maintained by the PTB -^^^^^^^^^^^^^^^^^^^^^^^^^ +Custom Workflows +^^^^^^^^^^^^^^^^ + +Custom workflows are project-owned. The two custom workflow forms are: + +* workflows seeded once by the PTB and then owned by the project +* workflow extensions that are enabled when the project adds the corresponding file + + +The PTB detects whether the relevant workflow file is present and activates a block +where is it called in corresponding calling workflow. If the file is absent, the PTB +does not add that block to the PTB-controlled workflows. + +For these custom workflows to work correctly, the reusable workflow must follow +the secret declaration format described in :ref:`custom_workflow_secrets` so the +calling PTB workflow can pass the required secrets through. + + +Workflows Initially Seeded by the PTB +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The PTB seeds these workflows for new projects, but after that the project owns them and PTB regeneration does not overwrite them. @@ -126,7 +149,7 @@ them and PTB regeneration does not overwrite them. - Runs long-running checks, which typically involve an Exasol database instance. Workflow Extensions -^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~ To use a workflow extension, a user must simply add the file to their project's ``.github/workflows`` directory. The PTB checks that this file exists, and if it does, diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index b1ee9ba5b..01f0ff384 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -19,6 +19,36 @@ standardized baseline that can be overridden in individual projects. :start-at: github_template_dict :end-before: @computed_field +.. _custom_workflow_secrets: + +Custom Workflow Secrets +^^^^^^^^^^^^^^^^^^^^^^^ + +The PTB extracts secret names from reusable custom workflow files and exposes them +through :py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict` under the +``custom_workflows`` entry. PTB-controlled workflow templates use those extracted +names when they call reusable workflows and forward secrets via ``secrets:``. + +To make a custom workflow compatible with this extraction, declare its secrets at the +top of the reusable workflow file under ``on.workflow_call`` and before ``jobs``. +The extractor reads that section and collects the secret names automatically. + +For example, ``slow-checks.yml`` can define its reusable workflow header like this: + +.. code-block:: yaml + + name: Slow-Checks + + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + SONAR_TOKEN: + required: true + +Those extracted secret names are then made available to the PTB templates that +reference the custom workflow. .. _workflow_matrix: diff --git a/doc/user_guide/troubleshooting/handle_zizmor_findings.rst b/doc/user_guide/troubleshooting/handle_zizmor_findings.rst index 69916e4fe..13bcba6c8 100644 --- a/doc/user_guide/troubleshooting/handle_zizmor_findings.rst +++ b/doc/user_guide/troubleshooting/handle_zizmor_findings.rst @@ -32,7 +32,11 @@ A typical line-level ignore looks like this: .. code-block:: yaml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet. + - name: Set up Poetry (${{ inputs.poetry-version }}) + shell: bash + run: | # zizmor: ignore[github-env] - This shared action is used by many workflows, and downstream steps need `poetry` on PATH; we do not have a safer replacement yet. + POETRY_VERSION="${INPUTS_POETRY_VERSION}" "$PYTHON_BINARY" "${{ github.action_path }}/ext/get_poetry.py" + echo "$HOME/.local/bin" >> $GITHUB_PATH Use configuration rules in ``.zizmor.yml`` only when the finding is genuinely project-wide. If you add a temporary rule while working through a batch of diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 4e7211342..de221b329 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -26,6 +26,9 @@ PLUGIN_ATTR_NAME, ) from exasol.toolbox.util.version import Version +from exasol.toolbox.util.workflows.custom_workflow_extractor import ( + CustomWorkflowExtractor, +) WORKFLOW_HEADER_PREFIX = ( "# Generated and maintained by the exasol-toolbox.\n" @@ -316,26 +319,19 @@ def github_template_dict(self) -> dict[str, Any]: Dictionary of variables to dynamically render Jinja2 templates into valid YAML configurations. """ - cd_extension = self.github_workflow_directory / "cd-extension.yml" - fast_tests_extension = ( - self.github_workflow_directory / "fast-tests-extension.yml" - ) - merge_gate_extension = ( - self.github_workflow_directory / "merge-gate-extension.yml" + custom_workflow_extractor = CustomWorkflowExtractor( + github_workflow_directory=self.github_workflow_directory, + sonar_token_name=self.sonar_token_name, ) return { + "custom_workflows": custom_workflow_extractor.build_custom_workflow_dict(), "dependency_manager_version": self.dependency_manager.version, "minimum_python_version": self.minimum_python_version, "os_version": self.os_version, "python_versions": self.python_versions, "sonar_token_name": self.sonar_token_name, "workflow_header": f"{WORKFLOW_HEADER_PREFIX}{__version__}.", - "workflow_extension": { - "cd": cd_extension.is_file(), - "fast_tests": fast_tests_extension.is_file(), - "merge_gate": merge_gate_extension.is_file(), - }, } @computed_field # type: ignore[misc] diff --git a/exasol/toolbox/templates/github/workflows/cd.yml b/exasol/toolbox/templates/github/workflows/cd.yml index d9238ce94..ca8ec258f 100644 --- a/exasol/toolbox/templates/github/workflows/cd.yml +++ b/exasol/toolbox/templates/github/workflows/cd.yml @@ -24,13 +24,18 @@ jobs: secrets: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - (% if workflow_extension.cd %) + (% if custom_workflows["cd-extension"].exists %) cd-extension: name: Extension uses: ./.github/workflows/cd-extension.yml needs: - check-release-tag - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if custom_workflows["cd-extension"].secrets %) + secrets: + (% for secret_name in custom_workflows["cd-extension"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: write (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/ci.yml b/exasol/toolbox/templates/github/workflows/ci.yml index c79cb52d6..c3c095328 100644 --- a/exasol/toolbox/templates/github/workflows/ci.yml +++ b/exasol/toolbox/templates/github/workflows/ci.yml @@ -9,7 +9,12 @@ jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if custom_workflows["merge-gate"].secrets %) + secrets: + (% for secret_name in custom_workflows["merge-gate"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read diff --git a/exasol/toolbox/templates/github/workflows/fast-tests.yml b/exasol/toolbox/templates/github/workflows/fast-tests.yml index 471e4532a..2f37f33b2 100644 --- a/exasol/toolbox/templates/github/workflows/fast-tests.yml +++ b/exasol/toolbox/templates/github/workflows/fast-tests.yml @@ -42,10 +42,16 @@ jobs: include-hidden-files: true overwrite: false -(% if workflow_extension.fast_tests %) +(% if custom_workflows["fast-tests-extension"].exists %) fast-tests-extension: name: Extension uses: ./.github/workflows/fast-tests-extension.yml + (% if custom_workflows["fast-tests-extension"].secrets %) + secrets: + (% for secret_name in custom_workflows["fast-tests-extension"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index c8a48483e..d68adf12c 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -3,6 +3,13 @@ name: Merge-Gate on: workflow_call: + (% if custom_workflows["merge-gate"].secrets %) + secrets: + (% for secret_name in custom_workflows["merge-gate"].secrets %) + (( secret_name )): + required: true + (% endfor %) + (% endif %) jobs: run-fast-checks: @@ -46,15 +53,25 @@ jobs: needs: - approve-run-slow-tests uses: ./.github/workflows/slow-checks.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if custom_workflows["slow-checks"].secrets %) + secrets: + (% for secret_name in custom_workflows["slow-checks"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read - (% if workflow_extension.merge_gate %) + (% if custom_workflows["merge-gate-extension"].exists %) merge-gate-extension: name: Extension uses: ./.github/workflows/merge-gate-extension.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if custom_workflows["merge-gate-extension"].secrets %) + secrets: + (% for secret_name in custom_workflows["merge-gate-extension"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% endif %) @@ -71,9 +88,9 @@ jobs: - run-fast-checks - run-fast-tests - run-slow-checks - (% if workflow_extension.merge_gate %) + (% if custom_workflows["merge-gate-extension"].exists %) - merge-gate-extension - (% endif %) + (% endif %) # To prevent accidentally merges, this step is required. For more details # see: https://github.com/exasol/python-toolbox/issues/563 steps: diff --git a/exasol/toolbox/templates/github/workflows/periodic-validation.yml b/exasol/toolbox/templates/github/workflows/periodic-validation.yml index 34f78370c..017f1c0fb 100644 --- a/exasol/toolbox/templates/github/workflows/periodic-validation.yml +++ b/exasol/toolbox/templates/github/workflows/periodic-validation.yml @@ -45,7 +45,12 @@ jobs: uses: ./.github/workflows/slow-checks.yml needs: - restrict-to-default-branch - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if custom_workflows["slow-checks"].secrets %) + secrets: + (% for secret_name in custom_workflows["slow-checks"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read diff --git a/exasol/toolbox/templates/github/workflows/slow-checks.yml b/exasol/toolbox/templates/github/workflows/slow-checks.yml index b15394bfa..487268d1b 100644 --- a/exasol/toolbox/templates/github/workflows/slow-checks.yml +++ b/exasol/toolbox/templates/github/workflows/slow-checks.yml @@ -21,8 +21,6 @@ jobs: runs-on: "(( os_version ))" permissions: contents: read - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} strategy: fail-fast: false matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} diff --git a/exasol/toolbox/util/workflows/custom_workflow.py b/exasol/toolbox/util/workflows/custom_workflow.py new file mode 100644 index 000000000..49c375b61 --- /dev/null +++ b/exasol/toolbox/util/workflows/custom_workflow.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from pathlib import Path + +from pydantic import ( + BaseModel, + ConfigDict, +) +from ruamel.yaml import ( + CommentedMap, +) + +from exasol.toolbox.util.workflows.render_yaml import parse_yaml_text + + +class CustomWorkflow(BaseModel): + """A project-owned workflow used for seeded workflows and extensions. + + These workflows are seeded by the PTB or extend PTB-provided workflows, but + they are maintained by the project itself rather than the PTB. See + `Custom Workflows `__. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + file_path: Path + yaml_content: CommentedMap + + @classmethod + def load_from_file(cls, file_path: Path) -> CustomWorkflow: + workflow_string = file_path.read_text() + yaml = parse_yaml_text(origin_path=file_path, workflow_string=workflow_string) + return cls(file_path=file_path, yaml_content=yaml) + + def extract_secrets(self) -> tuple[str, ...]: + """Return the secret names declared for ``workflow_call``. + + The reusable workflow must declare them near the top level of the file + like this: + + .. code-block:: yaml + + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + SONAR_TOKEN: + required: true + """ + workflow_call = self.yaml_content.get("on", {}).get("workflow_call") + if isinstance(workflow_call, dict): + if secrets := workflow_call.get("secrets", {}): + return tuple(secrets.keys()) + return () diff --git a/exasol/toolbox/util/workflows/custom_workflow_extractor.py b/exasol/toolbox/util/workflows/custom_workflow_extractor.py new file mode 100644 index 000000000..68f094614 --- /dev/null +++ b/exasol/toolbox/util/workflows/custom_workflow_extractor.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from pathlib import Path +from typing import TypedDict + +from pydantic import ( + BaseModel, + ConfigDict, +) + +from exasol.toolbox.util.workflows.custom_workflow import CustomWorkflow + + +class CustomWorkflowEntry(TypedDict): + exists: bool + secrets: tuple[str, ...] + + +class CustomWorkflowExtractor(BaseModel): + model_config = ConfigDict(frozen=True) + + github_workflow_directory: Path + sonar_token_name: str + custom_workflows: tuple[str, ...] = ( + "cd-extension", + "fast-tests-extension", + "merge-gate-extension", + "slow-checks", + ) + + def _build_custom_workflow_entry( + self, + workflow: str, + ) -> CustomWorkflowEntry: + file_path = self.github_workflow_directory / f"{workflow}.yml" + + secrets: tuple[str, ...] = () + if file_path.is_file(): + custom_workflow = CustomWorkflow.load_from_file(file_path=file_path) + secrets = custom_workflow.extract_secrets() + + return { + "exists": file_path.exists(), + "secrets": secrets, + } + + def _build_merge_gate_entry( + self, custom_workflows_dict: dict[str, CustomWorkflowEntry] + ) -> CustomWorkflowEntry: + return { + "exists": True, + "secrets": custom_workflows_dict["merge-gate-extension"]["secrets"] + + custom_workflows_dict["slow-checks"]["secrets"] + # from the `report.yml` + + (self.sonar_token_name,), + } + + def build_custom_workflow_dict( + self, + ) -> dict[str, CustomWorkflowEntry]: + """ + Build the template metadata used to specify whether a custom workflow is + present and which secrets its caller must pass through. + + The secret names are extracted from the reusable workflow itself via + :meth:`CustomWorkflow.extract_secrets`. + """ + + custom_workflows_dict = {} + for workflow in self.custom_workflows: + custom_workflows_dict[workflow] = self._build_custom_workflow_entry( + workflow=workflow, + ) + + custom_workflows_dict["merge-gate"] = self._build_merge_gate_entry( + custom_workflows_dict + ) + return custom_workflows_dict diff --git a/exasol/toolbox/util/workflows/exceptions.py b/exasol/toolbox/util/workflows/exceptions.py index b66acd3e6..1ec4b5479 100644 --- a/exasol/toolbox/util/workflows/exceptions.py +++ b/exasol/toolbox/util/workflows/exceptions.py @@ -31,7 +31,7 @@ class YamlOutputError(YamlError): class YamlParsingError(YamlError): """ - Raised when the rendered template is not a valid YAML file, as it cannot be + Raised when the loaded YAML is not a valid YAML file, as it cannot be parsed by ruamel-yaml. """ diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index 49a3f2696..3e02db891 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -37,6 +37,30 @@ ) +def get_standard_yaml() -> YAML: + """ + Prepare standard YAML class. + """ + yaml = YAML() + yaml.width = 200 + yaml.preserve_quotes = True + yaml.sort_base_mapping_type_on_output = False # type: ignore + yaml.indent(mapping=2, sequence=4, offset=2) + return yaml + + +def parse_yaml_text(origin_path: Path, workflow_string: str) -> CommentedMap: + """ + Parse YAML from text while keeping the origin path for diagnostics. + """ + try: + yaml = get_standard_yaml() + logger.debug("Parse %s with ruamel-yaml", origin_path) + return yaml.load(workflow_string) + except YAMLError as ex: + raise YamlParsingError(file_path=origin_path) from ex + + @dataclass(frozen=True) class YamlRenderer: """ @@ -49,18 +73,6 @@ class YamlRenderer: github_template_dict: dict[str, Any] file_path: Path - @staticmethod - def _get_standard_yaml() -> YAML: - """ - Prepare standard YAML class. - """ - yaml = YAML() - yaml.width = 200 - yaml.preserve_quotes = True - yaml.sort_base_mapping_type_on_output = False # type: ignore - yaml.indent(mapping=2, sequence=4, offset=2) - return yaml - def _render_with_jinja(self, input_str: str) -> str: """ Render the template with Jinja. @@ -81,19 +93,17 @@ def get_yaml_dict(self) -> CommentedMap: raw_content = self.file_path.read_text() try: workflow_string = self._render_with_jinja(raw_content) - yaml = self._get_standard_yaml() - logger.debug("Parse template with ruamel-yaml") - return yaml.load(workflow_string) except TemplateError as ex: raise TemplateRenderingError(file_path=self.file_path) from ex - except YAMLError as ex: - raise YamlParsingError(file_path=self.file_path) from ex + return parse_yaml_text( + origin_path=self.file_path, workflow_string=workflow_string + ) def get_as_string(self, yaml_dict: CommentedMap) -> str: """ Output a YAML string. """ - yaml = self._get_standard_yaml() + yaml = get_standard_yaml() try: logger.debug("Output workflow as string") with io.StringIO() as stream: diff --git a/exasol/toolbox/util/workflows/workflow.py b/exasol/toolbox/util/workflows/workflow.py index 9d96293be..8b6d73199 100644 --- a/exasol/toolbox/util/workflows/workflow.py +++ b/exasol/toolbox/util/workflows/workflow.py @@ -26,6 +26,12 @@ class Workflow(BaseModel): + """A PTB-maintained GitHub workflow rendered from a workflow template. + + These workflows are provided and maintained by the PTB. See + `Maintained by the PTB `__. + """ + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) template_path: Path diff --git a/test/unit/config_test.py b/test/unit/config_test.py index 3115d0632..77a370253 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -49,10 +49,16 @@ def test_works_as_defined(tmp_path, test_project_config_factory): "github_workflow_directory": tmp_path / ".github" / "workflows", "github_workflow_patcher_yaml": None, "github_template_dict": { + "custom_workflows": { + "cd-extension": {"exists": False, "secrets": ()}, + "fast-tests-extension": {"exists": False, "secrets": ()}, + "merge-gate-extension": {"exists": False, "secrets": ()}, + "slow-checks": {"exists": False, "secrets": ()}, + "merge-gate": {"exists": True, "secrets": ("SONAR_TOKEN",)}, + }, "dependency_manager_version": "2.3.0", "minimum_python_version": "3.10", "os_version": "ubuntu-24.04", - "sonar_token_name": "SONAR_TOKEN", "python_versions": ( "3.10", "3.11", @@ -60,11 +66,7 @@ def test_works_as_defined(tmp_path, test_project_config_factory): "3.13", "3.14", ), - "workflow_extension": { - "cd": False, - "fast_tests": False, - "merge_gate": False, - }, + "sonar_token_name": "SONAR_TOKEN", }, "minimum_exasol_version": "8.29.13", "minimum_python_version": "3.10", diff --git a/test/unit/util/workflows/custom_workflow_extractor_test.py b/test/unit/util/workflows/custom_workflow_extractor_test.py new file mode 100644 index 000000000..53e43e9f1 --- /dev/null +++ b/test/unit/util/workflows/custom_workflow_extractor_test.py @@ -0,0 +1,149 @@ +from inspect import cleandoc + +import pytest + +from exasol.toolbox.util.workflows.custom_workflow_extractor import ( + CustomWorkflowExtractor, +) + + +@pytest.fixture +def workflow_directory(tmp_path): + workflow_directory = tmp_path / ".github" / "workflows" + workflow_directory.mkdir(parents=True) + return workflow_directory + + +@pytest.fixture +def custom_workflow_extractor(workflow_directory): + return CustomWorkflowExtractor( + github_workflow_directory=workflow_directory, + sonar_token_name="SONAR_TOKEN", + ) + + +class TestBuildCustomWorkflowEntry: + @staticmethod + def test_file_does_not_exist(custom_workflow_extractor): + custom_workflow_entry = custom_workflow_extractor._build_custom_workflow_entry( + workflow="slow-checks" + ) + + assert custom_workflow_entry == {"exists": False, "secrets": ()} + + @staticmethod + def test_file_does_not_contain_secrets( + custom_workflow_extractor, workflow_directory + ): + workflow = "slow-checks" + workflow_path = workflow_directory / f"{workflow}.yml" + workflow_path.write_text(cleandoc(""" + name: Slow-Checks + + on: + workflow_call: + """)) + + custom_workflow_entry = custom_workflow_extractor._build_custom_workflow_entry( + workflow=workflow + ) + + assert custom_workflow_entry == {"exists": True, "secrets": ()} + + @staticmethod + def test_file_contains_secret(custom_workflow_extractor, workflow_directory): + workflow = "slow-checks" + workflow_path = workflow_directory / f"{workflow}.yml" + workflow_path.write_text(cleandoc(""" + name: Slow-Checks + + on: + workflow_call: + secrets: + SLOW_CHECK_SECRET: + required: true + """)) + + custom_workflow_entry = custom_workflow_extractor._build_custom_workflow_entry( + workflow=workflow + ) + + assert custom_workflow_entry == { + "exists": True, + "secrets": ("SLOW_CHECK_SECRET",), + } + + +class TestBuildMergeGateEntry: + @staticmethod + def test_build_merge_gate_entry(custom_workflow_extractor): + custom_workflows_dict = { + "merge-gate-extension": {"exists": True, "secrets": ("EXT_SECRET",)}, + "slow-checks": {"exists": True, "secrets": ("SLOW_SECRET",)}, + } + + merge_gate_entry = custom_workflow_extractor._build_merge_gate_entry( + custom_workflows_dict + ) + + assert merge_gate_entry == { + "exists": True, + "secrets": ("EXT_SECRET", "SLOW_SECRET", "SONAR_TOKEN"), + } + + +class TestBuildCustomWorkflowDict: + default_custom_workflow_dict = { + "cd-extension": {"exists": False, "secrets": ()}, + "fast-tests-extension": {"exists": False, "secrets": ()}, + "merge-gate-extension": {"exists": False, "secrets": ()}, + "slow-checks": {"exists": False, "secrets": ()}, + "merge-gate": {"exists": True, "secrets": ("SONAR_TOKEN",)}, + } + + def test_no_custom_workflows_exist(self, custom_workflow_extractor): + custom_workflow_dict = custom_workflow_extractor.build_custom_workflow_dict() + + assert custom_workflow_dict == self.default_custom_workflow_dict + + @pytest.mark.parametrize( + "workflow", + ( + "cd-extension", + "fast-tests-extension", + "merge-gate-extension", + "slow-checks", + ), + ) + def test_custom_workflow_written_to( + self, + custom_workflow_extractor, + workflow_directory, + workflow, + ): + secret = f"{workflow.upper()}_SECRET" + workflow_path = workflow_directory / f"{workflow}.yml" + workflow_path.write_text(cleandoc(f""" + name: {workflow} + + on: + workflow_call: + secrets: + {secret}: + required: true + """)) + + custom_workflow_dict = custom_workflow_extractor.build_custom_workflow_dict() + + expected_custom_workflow_dict = self.default_custom_workflow_dict.copy() + expected_custom_workflow_dict[workflow] = custom_workflow_dict[workflow] = { + "exists": True, + "secrets": (secret,), + } + if workflow in ("merge-gate-extension", "slow-checks"): + expected_custom_workflow_dict["merge-gate"]["secrets"] = ( + secret, + "SONAR_TOKEN", + ) + + assert custom_workflow_dict == expected_custom_workflow_dict diff --git a/test/unit/util/workflows/custom_workflow_test.py b/test/unit/util/workflows/custom_workflow_test.py new file mode 100644 index 000000000..14fc74b53 --- /dev/null +++ b/test/unit/util/workflows/custom_workflow_test.py @@ -0,0 +1,55 @@ +from inspect import cleandoc +from pathlib import Path + +import pytest + +from exasol.toolbox.util.workflows.custom_workflow_extractor import CustomWorkflow + + +@pytest.fixture +def test_yml(tmp_path: Path) -> Path: + return tmp_path / "test.yml" + + +class TestCustomWorkflow: + yaml_with_secrets = cleandoc(""" + name: Build & Publish + + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + SONAR_TOKEN: + required: true + """) + + yaml_without_secrets = cleandoc(""" + name: Build & Publish + + on: + workflow_call: + """) + + def test_load_from_file(self, test_yml): + test_yml.write_text(self.yaml_with_secrets) + + custom_workflow = CustomWorkflow.load_from_file(file_path=test_yml) + + assert custom_workflow.yaml_content + + def test_extract_secrets_when_present(self, test_yml): + test_yml.write_text(self.yaml_with_secrets) + + custom_workflow = CustomWorkflow.load_from_file(file_path=test_yml) + secrets = custom_workflow.extract_secrets() + + assert secrets == ("PYPI_TOKEN", "SONAR_TOKEN") + + def test_extract_secrets_when_no_secrets_present(self, test_yml): + test_yml.write_text(self.yaml_without_secrets) + + custom_workflow = CustomWorkflow.load_from_file(file_path=test_yml) + secrets = custom_workflow.extract_secrets() + + assert secrets == () diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index 92eb804fc..20f6164f9 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -16,6 +16,7 @@ ) from exasol.toolbox.util.workflows.render_yaml import ( YamlRenderer, + parse_yaml_text, ) @@ -177,7 +178,67 @@ def test_preserves_list_format(test_yml, yaml_renderer): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(input_yaml) @staticmethod - def test_parsing_fails_when_yaml_malformed(test_yml, yaml_renderer): + def test_yaml_cannot_output_to_string(test_yml, yaml_renderer): + input_yaml = """ + steps: + # Comment in nested area + - name: SCM Checkout # Comment inline + uses: actions/checkout@v6 + # Comment in step + """ + content = cleandoc(input_yaml) + test_yml.write_text(content) + + yaml_dict = yaml_renderer.get_yaml_dict() + yaml_dict["steps"][0]["name"] = lambda x: x + 1 + + with pytest.raises(YamlOutputError, match="could not be output") as ex: + yaml_renderer.get_as_string(yaml_dict) + + assert isinstance(ex.value.__cause__, RepresenterError) + assert "cannot represent an object" in str(ex.value.__cause__) + + +class TestParseYamlText: + @staticmethod + def test_parsing_succeeds(tmp_path): + input_yaml = """ + name: Build & Publish + + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + + jobs: + cd-job: + name: Continuous Delivery + permissions: + contents: write + """ + + yaml_dict = parse_yaml_text( + origin_path=tmp_path / "dummy.yml", workflow_string=cleandoc(input_yaml) + ) + + assert yaml_dict == { + "name": "Build & Publish", + "on": { + "workflow_call": { + "secrets": {"PYPI_TOKEN": {"required": True}}, + } + }, + "jobs": { + "cd-job": { + "name": "Continuous Delivery", + "permissions": {"contents": "write"}, + } + }, + } + + @staticmethod + def test_parsing_fails_when_yaml_malformed(tmp_path): bad_template = """ name: Publish Documentation @@ -195,37 +256,18 @@ def test_parsing_fails_when_yaml_malformed(test_yml, yaml_renderer): - name: SCM Checkout uses: actions/checkout@v5 """ - test_yml.write_text(cleandoc(bad_template)) with pytest.raises( YamlParsingError, match="Check for invalid YAML syntax." ) as ex: - yaml_renderer.get_yaml_dict() + parse_yaml_text( + origin_path=tmp_path / "dummy.yml", + workflow_string=cleandoc(bad_template), + ) assert isinstance(ex.value.__cause__, ParserError) assert "while parsing a block collection" in str(ex.value.__cause__) - @staticmethod - def test_yaml_cannot_output_to_string(test_yml, yaml_renderer): - input_yaml = """ - steps: - # Comment in nested area - - name: SCM Checkout # Comment inline - uses: actions/checkout@v6 - # Comment in step - """ - content = cleandoc(input_yaml) - test_yml.write_text(content) - - yaml_dict = yaml_renderer.get_yaml_dict() - yaml_dict["steps"][0]["name"] = lambda x: x + 1 - - with pytest.raises(YamlOutputError, match="could not be output") as ex: - yaml_renderer.get_as_string(yaml_dict) - - assert isinstance(ex.value.__cause__, RepresenterError) - assert "cannot represent an object" in str(ex.value.__cause__) - class TestYamlRendererJinja: @staticmethod @@ -252,7 +294,9 @@ def test_updates_jinja_variables(test_yml, yaml_renderer): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_does_not_add_jinja_block(test_yml, yaml_renderer, project_config): + def test_omits_block_when_extension_is_missing( + test_yml, yaml_renderer, project_config + ): input_yaml = """ jobs: run-unit-tests: @@ -270,10 +314,16 @@ def test_does_not_add_jinja_block(test_yml, yaml_renderer, project_config): id: check-out-repository uses: actions/checkout@v6 - (% if workflow_extension.fast_tests %) + (% if custom_workflows["fast-tests-extension"].exists %) fast-tests-extension: name: Extension uses: ./.github/workflows/fast-tests-extension.yml + (% if custom_workflows["fast-tests-extension"].secrets %) + secrets: + (% for secret_name in custom_workflows["fast-tests-extension"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% endif %) @@ -305,7 +355,7 @@ def test_does_not_add_jinja_block(test_yml, yaml_renderer, project_config): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_adds_jinja_block(test_yml, project_config): + def test_includes_if_block_when_extension_is_present(test_yml, project_config): input_yaml = """ jobs: run-unit-tests: @@ -323,10 +373,16 @@ def test_adds_jinja_block(test_yml, project_config): id: check-out-repository uses: actions/checkout@v6 - (% if workflow_extension.fast_tests %) + (% if custom_workflows["fast-tests-extension"].exists %) fast-tests-extension: name: Extension uses: ./.github/workflows/fast-tests-extension.yml + (% if custom_workflows["fast-tests-extension"].secrets %) + secrets: + (% for secret_name in custom_workflows["fast-tests-extension"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% endif %) @@ -352,13 +408,23 @@ def test_adds_jinja_block(test_yml, project_config): fast-tests-extension: name: Extension uses: ./.github/workflows/fast-tests-extension.yml + secrets: + FAST_TEST_SECRET: ${{ secrets.FAST_TEST_SECRET }} permissions: contents: read """ workflow_directory = project_config.github_workflow_directory workflow_directory.mkdir(parents=True) - (workflow_directory / "fast-tests-extension.yml").touch() + (workflow_directory / "fast-tests-extension.yml").write_text(cleandoc(""" + name: Fast-Tests-Extension + + on: + workflow_call: + secrets: + FAST_TEST_SECRET: + required: true + """)) content = cleandoc(input_yaml) test_yml.write_text(content) @@ -371,6 +437,90 @@ def test_adds_jinja_block(test_yml, project_config): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod + def test_includes_extension_with_multiple_secrets(test_yml, project_config): + input_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "(( os_version ))" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: (( python_versions | tojson )) + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + (% if custom_workflows["merge-gate"].exists %) + merge-gate-extension: + uses: ./.github/workflows/merge-gate-extension.yml + (% if custom_workflows["merge-gate-extension"].secrets %) + secrets: + (% for secret_name in custom_workflows["merge-gate-extension"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) + permissions: + contents: read + (% endif %) + + """ + expected_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "ubuntu-24.04" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + merge-gate-extension: + uses: ./.github/workflows/merge-gate-extension.yml + secrets: + MERGE_GATE_SECRET: ${{ secrets.MERGE_GATE_SECRET }} + ANOTHER_SECRET: ${{ secrets.ANOTHER_SECRET }} + permissions: + contents: read + + """ + workflow_directory = project_config.github_workflow_directory + workflow_directory.mkdir(parents=True) + (workflow_directory / "merge-gate-extension.yml").write_text(cleandoc(""" + name: Merge-Gate-Extension + + on: + workflow_call: + secrets: + MERGE_GATE_SECRET: + required: true + ANOTHER_SECRET: + required: true + """)) + + content = cleandoc(input_yaml) + test_yml.write_text(content) + + yaml_renderer = YamlRenderer( + github_template_dict=project_config.github_template_dict, + file_path=test_yml, + ) + + yaml_dict = yaml_renderer.get_yaml_dict() + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod def test_jinja_variable_unknown(test_yml, yaml_renderer): input_yaml = """