From be18a308e006f0b0af1431058724d257160528ac Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 13:33:27 +0200 Subject: [PATCH 01/30] Remove as no secret needed here --- exasol/toolbox/templates/github/workflows/slow-checks.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/exasol/toolbox/templates/github/workflows/slow-checks.yml b/exasol/toolbox/templates/github/workflows/slow-checks.yml index b15394bfa..487268d1b 100644 --- a/exasol/toolbox/templates/github/workflows/slow-checks.yml +++ b/exasol/toolbox/templates/github/workflows/slow-checks.yml @@ -21,8 +21,6 @@ jobs: runs-on: "(( os_version ))" permissions: contents: read - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} strategy: fail-fast: false matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} From 14b547cf4d0fa19fa440cb4f4a6fcc0e1f7cfbbc Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 13:33:41 +0200 Subject: [PATCH 02/30] Add initial slow-checks secret modifier --- .github/workflows/merge-gate.yml | 1 - .github/workflows/periodic-validation.yml | 1 - exasol/toolbox/config.py | 10 ++++++++++ .../toolbox/templates/github/workflows/merge-gate.yml | 7 ++++++- .../templates/github/workflows/periodic-validation.yml | 7 ++++++- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 39fb7ee9d..aa3e3e29b 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -47,7 +47,6 @@ jobs: needs: - approve-run-slow-tests uses: ./.github/workflows/slow-checks.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. permissions: contents: read diff --git a/.github/workflows/periodic-validation.yml b/.github/workflows/periodic-validation.yml index 01b25f5b3..62834750f 100644 --- a/.github/workflows/periodic-validation.yml +++ b/.github/workflows/periodic-validation.yml @@ -46,7 +46,6 @@ jobs: uses: ./.github/workflows/slow-checks.yml needs: - restrict-to-default-branch - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. permissions: contents: read diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 4e7211342..fccfad0a3 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -207,6 +207,12 @@ class BaseConfig(BaseModel): are supported. """, ) + secrets_slow_checks: tuple[str, ...] = Field( + default=(), + description="""This tuple defines the string names of secrets needed to pass + to the slow-checks.yml. + """ + ) @computed_field # type: ignore[misc] @property @@ -336,6 +342,10 @@ def github_template_dict(self) -> dict[str, Any]: "fast_tests": fast_tests_extension.is_file(), "merge_gate": merge_gate_extension.is_file(), }, + "secrets": + { + "slow_checks": self.secrets_slow_checks, + } } @computed_field # type: ignore[misc] diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index a7498dd92..cdd3f60f9 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -46,7 +46,12 @@ jobs: needs: - approve-run-slow-tests uses: ./.github/workflows/slow-checks.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.slow_checks %) + secrets: + (% for secret_name in secrets.slow_checks %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read diff --git a/exasol/toolbox/templates/github/workflows/periodic-validation.yml b/exasol/toolbox/templates/github/workflows/periodic-validation.yml index 34f78370c..99368728f 100644 --- a/exasol/toolbox/templates/github/workflows/periodic-validation.yml +++ b/exasol/toolbox/templates/github/workflows/periodic-validation.yml @@ -45,7 +45,12 @@ jobs: uses: ./.github/workflows/slow-checks.yml needs: - restrict-to-default-branch - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.slow_checks %) + secrets: + (% for secret_name in secrets.slow_checks %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read From 0a5e2053a7ce752ac8c021b46f78d225cd08153c Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 14:23:42 +0200 Subject: [PATCH 03/30] Switch to supporting multiple customizations --- .github/workflows/ci.yml | 3 +- .github/workflows/merge-gate.yml | 1 - exasol/toolbox/config.py | 47 +++++++++++++++---- .../toolbox/templates/github/workflows/cd.yml | 7 ++- .../toolbox/templates/github/workflows/ci.yml | 7 ++- .../templates/github/workflows/merge-gate.yml | 7 ++- test/unit/config_test.py | 11 +++++ 7 files changed, 69 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e80c736c0..0a34a268b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,8 @@ jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} permissions: contents: read diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index aa3e3e29b..408033e39 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -52,7 +52,6 @@ jobs: merge-gate-extension: uses: ./.github/workflows/merge-gate-extension.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. permissions: contents: read diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index fccfad0a3..3af13de62 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -137,6 +137,35 @@ def check_minimum_version(cls, v: str, info: ValidationInfo) -> str: return v +class CustomWorkflowSecrets(BaseModel): + cd_extension: tuple[str, ...] = Field( + default=(), + description=""" + This tuple defines the string names of secrets needed to pass to the + cd-extension.yml. + """, + ) + slow_checks: tuple[str, ...] = Field( + default=(), + description=""" + This tuple defines the string names of secrets needed to pass to the + slow_checks.yml. + """, + ) + merge_gate_extension: tuple[str, ...] = Field( + default=(), + description=""" + This tuple defines the string names of secrets needed to pass to the + merge-gate-extension.yml. + """, + ) + + def get_secrets_dict(self) -> dict[str, tuple[str, ...]]: + secrets = self.model_dump(exclude_computed_fields=True) + secrets["merge_gate"] = self.merge_gate_extension + self.slow_checks + return secrets + + class BaseConfig(BaseModel): """ Basic configuration for projects using the PTB @@ -207,11 +236,11 @@ class BaseConfig(BaseModel): are supported. """, ) - secrets_slow_checks: tuple[str, ...] = Field( - default=(), - description="""This tuple defines the string names of secrets needed to pass - to the slow-checks.yml. - """ + custom_workflow_secrets: CustomWorkflowSecrets = Field( + default=CustomWorkflowSecrets(), + description=""" + This object is used to set the secret arrays for custom workflows. + """, ) @computed_field # type: ignore[misc] @@ -330,11 +359,15 @@ def github_template_dict(self) -> dict[str, Any]: self.github_workflow_directory / "merge-gate-extension.yml" ) + secrets = self.custom_workflow_secrets.get_secrets_dict() + secrets["merge_gate"] += (self.sonar_token_name,) + return { "dependency_manager_version": self.dependency_manager.version, "minimum_python_version": self.minimum_python_version, "os_version": self.os_version, "python_versions": self.python_versions, + "secrets": secrets, "sonar_token_name": self.sonar_token_name, "workflow_header": f"{WORKFLOW_HEADER_PREFIX}{__version__}.", "workflow_extension": { @@ -342,10 +375,6 @@ def github_template_dict(self) -> dict[str, Any]: "fast_tests": fast_tests_extension.is_file(), "merge_gate": merge_gate_extension.is_file(), }, - "secrets": - { - "slow_checks": self.secrets_slow_checks, - } } @computed_field # type: ignore[misc] diff --git a/exasol/toolbox/templates/github/workflows/cd.yml b/exasol/toolbox/templates/github/workflows/cd.yml index 618063c3e..45b77545b 100644 --- a/exasol/toolbox/templates/github/workflows/cd.yml +++ b/exasol/toolbox/templates/github/workflows/cd.yml @@ -29,7 +29,12 @@ jobs: uses: ./.github/workflows/cd-extension.yml needs: - check-release-tag - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.cd_extension %) + secrets: + (% for secret_name in secrets.cd_extension %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: write (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/ci.yml b/exasol/toolbox/templates/github/workflows/ci.yml index c79cb52d6..34c2424e2 100644 --- a/exasol/toolbox/templates/github/workflows/ci.yml +++ b/exasol/toolbox/templates/github/workflows/ci.yml @@ -9,7 +9,12 @@ jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.merge_gate %) + secrets: + (% for secret_name in secrets.merge_gate %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index cdd3f60f9..459ac5397 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -58,7 +58,12 @@ jobs: (% if workflow_extension.merge_gate %) merge-gate-extension: uses: ./.github/workflows/merge-gate-extension.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.merge_gate_extension %) + secrets: + (% for secret_name in secrets.merge_gate_extension %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% endif %) diff --git a/test/unit/config_test.py b/test/unit/config_test.py index 3115d0632..f30fae381 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -42,6 +42,11 @@ def test_works_as_defined(tmp_path, test_project_config_factory): assert config_dump == { "add_to_excluded_python_paths": (), "create_major_version_tags": False, + "custom_workflow_secrets": { + "cd_extension": (), + "merge_gate_extension": (), + "slow_checks": (), + }, "dependency_manager": {"name": "poetry", "version": "2.3.0"}, "documentation_path": root_path / "doc", "exasol_versions": ("8.29.13", "2025.1.8"), @@ -60,6 +65,12 @@ def test_works_as_defined(tmp_path, test_project_config_factory): "3.13", "3.14", ), + "secrets": { + "cd_extension": (), + "merge_gate": ("SONAR_TOKEN",), + "merge_gate_extension": (), + "slow_checks": (), + }, "workflow_extension": { "cd": False, "fast_tests": False, From c7cf8dbf7ba6d9245fd7670d7ecd760a3dadaa16 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 14:24:56 +0200 Subject: [PATCH 04/30] Switch example as secrets allowed --- doc/user_guide/troubleshooting/handle_zizmor_findings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user_guide/troubleshooting/handle_zizmor_findings.rst b/doc/user_guide/troubleshooting/handle_zizmor_findings.rst index 69916e4fe..4a3efb191 100644 --- a/doc/user_guide/troubleshooting/handle_zizmor_findings.rst +++ b/doc/user_guide/troubleshooting/handle_zizmor_findings.rst @@ -32,7 +32,7 @@ A typical line-level ignore looks like this: .. code-block:: yaml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet. + secrets: inherit # zizmor: ignore[github-env] - This shared action is used by many workflows, and downstream steps need `poetry` on PATH; we do not have a safer replacement yet. Use configuration rules in ``.zizmor.yml`` only when the finding is genuinely project-wide. If you add a temporary rule while working through a batch of From d6970d0d355b55436f35fec02dd10dc1ec2c15cb Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 14:33:03 +0200 Subject: [PATCH 05/30] Add unit test for CustomWorkflowSecrets --- test/unit/config_test.py | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/test/unit/config_test.py b/test/unit/config_test.py index f30fae381..df401743e 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -10,6 +10,7 @@ from exasol.toolbox.config import ( DEFAULT_EXCLUDED_PATHS, BaseConfig, + CustomWorkflowSecrets, DependencyManager, minimum_declared_version, valid_version_string, @@ -143,6 +144,60 @@ def sonar_token_name(self) -> str: return "SONAR_ANOTHER_TOKEN" +class TestCustomWorkflowSecrets: + @staticmethod + def test_default(): + custom_workflow_secrets = CustomWorkflowSecrets() + secrets = custom_workflow_secrets.get_secrets_dict() + + assert secrets == { + "cd_extension": (), + "merge_gate": (), + "merge_gate_extension": (), + "slow_checks": (), + } + + @staticmethod + def test_single_override(): + cd_ext_secret = "CD_SECRET" + + custom_workflow_secrets = CustomWorkflowSecrets( + cd_extension=(cd_ext_secret,), + ) + secrets = custom_workflow_secrets.get_secrets_dict() + + assert secrets == { + "cd_extension": (cd_ext_secret,), + "merge_gate": (), + "merge_gate_extension": (), + "slow_checks": (), + } + + @staticmethod + def test_multiple_overrides(): + cd_ext_secret = "CD_SECRET" + merge_gate_ext_secret = "MERGE_GATE_SECRET" + slow_checks_secret = "SLOW_CHECKS_SECRET" + + custom_workflow_secrets = CustomWorkflowSecrets( + cd_extension=(cd_ext_secret,), + merge_gate_extension=(merge_gate_ext_secret, merge_gate_ext_secret), + slow_checks=(slow_checks_secret,), + ) + secrets = custom_workflow_secrets.get_secrets_dict() + + assert secrets == { + "cd_extension": (cd_ext_secret,), + "merge_gate": ( + merge_gate_ext_secret, + merge_gate_ext_secret, + slow_checks_secret, + ), + "merge_gate_extension": (merge_gate_ext_secret, merge_gate_ext_secret), + "slow_checks": (slow_checks_secret,), + } + + def test_expansion_validation_fails_for_invalid_version(): with pytest.raises(ValueError): BaseConfigExpansion(python_versions=("1.f.0",)) From 84550b41e8deb19d96efc9b68ca5238a8ef24900 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 14:34:45 +0200 Subject: [PATCH 06/30] Add comment --- exasol/toolbox/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 3af13de62..2e708a107 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -360,6 +360,7 @@ def github_template_dict(self) -> dict[str, Any]: ) secrets = self.custom_workflow_secrets.get_secrets_dict() + # merge-gate.yml also calls report.yml and needs Sonar token secrets["merge_gate"] += (self.sonar_token_name,) return { From f595f3e13c7961e86658d817e4af74c6dfd539de Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 14:43:52 +0200 Subject: [PATCH 07/30] Update tests to include checking when has the jinja secrets --- test/unit/config_test.py | 6 +- test/unit/util/workflows/render_yaml_test.py | 95 +++++++++++++++++++- 2 files changed, 96 insertions(+), 5 deletions(-) diff --git a/test/unit/config_test.py b/test/unit/config_test.py index df401743e..4b7be1c3f 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -146,7 +146,7 @@ def sonar_token_name(self) -> str: class TestCustomWorkflowSecrets: @staticmethod - def test_default(): + def test_get_secrets_dict_defaults(): custom_workflow_secrets = CustomWorkflowSecrets() secrets = custom_workflow_secrets.get_secrets_dict() @@ -158,7 +158,7 @@ def test_default(): } @staticmethod - def test_single_override(): + def test_get_secrets_dict_with_cd_extension_override(): cd_ext_secret = "CD_SECRET" custom_workflow_secrets = CustomWorkflowSecrets( @@ -174,7 +174,7 @@ def test_single_override(): } @staticmethod - def test_multiple_overrides(): + def test_get_secrets_dict_merges_merge_gate_extension_and_slow_checks(): cd_ext_secret = "CD_SECRET" merge_gate_ext_secret = "MERGE_GATE_SECRET" slow_checks_secret = "SLOW_CHECKS_SECRET" diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index eed3331ff..a48ff4968 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -252,7 +252,9 @@ def test_updates_jinja_variables(test_yml, yaml_renderer): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_does_not_add_jinja_block(test_yml, yaml_renderer, project_config): + def test_omits_block_when_extension_is_missing( + test_yml, yaml_renderer, project_config + ): input_yaml = """ jobs: run-unit-tests: @@ -304,7 +306,9 @@ def test_does_not_add_jinja_block(test_yml, yaml_renderer, project_config): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_adds_jinja_block(test_yml, project_config): + def test_includes_if_block_when_extension_is_present( + test_yml, project_config + ): input_yaml = """ jobs: run-unit-tests: @@ -368,6 +372,93 @@ def test_adds_jinja_block(test_yml, project_config): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod + def test_includes_extension_with_multiple_secrets( + test_yml, project_config + ): + input_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "(( os_version ))" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: (( python_versions | tojson )) + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + (% if workflow_extension.merge_gate %) + merge-gate-extension: + uses: ./.github/workflows/merge-gate-extension.yml + (% if secrets.merge_gate_extension %) + secrets: + (% for secret_name in secrets.merge_gate_extension %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) + permissions: + contents: read + (% endif %) + + """ + expected_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "ubuntu-24.04" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + merge-gate-extension: + uses: ./.github/workflows/merge-gate-extension.yml + secrets: + MERGE_GATE_SECRET: ${{ secrets.MERGE_GATE_SECRET }} + ANOTHER_SECRET: ${{ secrets.ANOTHER_SECRET }} + permissions: + contents: read + + """ + workflow_directory = project_config.github_workflow_directory + workflow_directory.mkdir(parents=True) + (workflow_directory / "merge-gate-extension.yml").touch() + + custom_workflow_secrets = project_config.custom_workflow_secrets.model_copy( + update={ + "merge_gate_extension": ( + "MERGE_GATE_SECRET", + "ANOTHER_SECRET", + ) + } + ) + updated_project_config = project_config.model_copy( + update={"custom_workflow_secrets": custom_workflow_secrets} + ) + yaml_renderer = YamlRenderer( + github_template_dict=updated_project_config.github_template_dict, + file_path=test_yml, + ) + + content = cleandoc(input_yaml) + test_yml.write_text(content) + + yaml_dict = yaml_renderer.get_yaml_dict() + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod def test_jinja_variable_unknown(test_yml, yaml_renderer): input_yaml = """ From 6c1a4f14fcfa1f6cbd44154100db796dcc01be49 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 14:55:34 +0200 Subject: [PATCH 08/30] Add documentation for new feature --- doc/api/custom_workflow_secrets.rst | 12 ++++++++++++ doc/api/index.rst | 1 + .../features/github_workflows/workflow_variables.rst | 9 +++++++++ 3 files changed, 22 insertions(+) create mode 100644 doc/api/custom_workflow_secrets.rst diff --git a/doc/api/custom_workflow_secrets.rst b/doc/api/custom_workflow_secrets.rst new file mode 100644 index 000000000..5d58289d7 --- /dev/null +++ b/doc/api/custom_workflow_secrets.rst @@ -0,0 +1,12 @@ +.. _custom_workflow_secrets: + +CustomWorkflowSecrets +===================== + +.. currentmodule:: exasol.toolbox.config + +.. autoclass:: CustomWorkflowSecrets + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource diff --git a/doc/api/index.rst b/doc/api/index.rst index 8ad43c961..90e79ce62 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -7,5 +7,6 @@ :maxdepth: 2 base_config + custom_workflow_secrets workflow_exceptions workflow_patcher_config diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index b1ee9ba5b..182bbb593 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -19,6 +19,15 @@ standardized baseline that can be overridden in individual projects. :start-at: github_template_dict :end-before: @computed_field +Custom Workflow Secrets +^^^^^^^^^^^^^^^^^^^^^^^ + +If your project needs to pass secrets into project-controlled workflows, configure +the ``custom_workflow_secrets`` field on :class:`exasol.toolbox.config.BaseConfig`. +That field uses :class:`exasol.toolbox.config.CustomWorkflowSecrets` and lets you +define separate secret tuples for specific workflows. See the API reference for +:class:`exasol.toolbox.config.CustomWorkflowSecrets` for the exact structure. + .. _workflow_matrix: From e325ba6e3bf38fdc5e309f988298790f833624fe Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 14:56:03 +0200 Subject: [PATCH 09/30] Alphabetize by name --- exasol/toolbox/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 2e708a107..822f1b350 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -145,18 +145,18 @@ class CustomWorkflowSecrets(BaseModel): cd-extension.yml. """, ) - slow_checks: tuple[str, ...] = Field( + merge_gate_extension: tuple[str, ...] = Field( default=(), description=""" This tuple defines the string names of secrets needed to pass to the - slow_checks.yml. + merge-gate-extension.yml. """, ) - merge_gate_extension: tuple[str, ...] = Field( + slow_checks: tuple[str, ...] = Field( default=(), description=""" This tuple defines the string names of secrets needed to pass to the - merge-gate-extension.yml. + slow_checks.yml. """, ) From 178d2bcd276d46ede7837a2e9bce2d9b939e3005 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 15:09:16 +0200 Subject: [PATCH 10/30] Add missing needed block --- .github/workflows/merge-gate.yml | 3 +++ .../toolbox/templates/github/workflows/merge-gate.yml | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 408033e39..b77cc4bab 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -4,6 +4,9 @@ name: Merge-Gate on: workflow_call: + secrets: + SONAR_TOKEN: + required: true jobs: run-fast-checks: diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index 459ac5397..ba08e8046 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -3,6 +3,13 @@ name: Merge-Gate on: workflow_call: + (% if secrets.merge_gate %) + secrets: + (% for secret_name in secrets.merge_gate %) + (( secret_name )): + required: true + (% endfor %) + (% endif %) jobs: run-fast-checks: @@ -80,9 +87,9 @@ jobs: - run-fast-checks - run-fast-tests - run-slow-checks - (% if workflow_extension.merge_gate %) + (% if workflow_extension.merge_gate %) - merge-gate-extension - (% endif %) + (% endif %) # To prevent accidentally merges, this step is required. For more details # see: https://github.com/exasol/python-toolbox/issues/563 steps: From abe51a89732872b4c275ac328a93fe52ec847eba Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 15:13:58 +0200 Subject: [PATCH 11/30] Update documentation --- .../github_workflows/workflow_variables.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index 182bbb593..f509cc284 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -28,6 +28,24 @@ That field uses :class:`exasol.toolbox.config.CustomWorkflowSecrets` and lets yo define separate secret tuples for specific workflows. See the API reference for :class:`exasol.toolbox.config.CustomWorkflowSecrets` for the exact structure. +Those custom secrets must be declared at the top of the reusable workflow file, under +``on.workflow_call`` and before ``jobs``. The generated workflows rely on that shape +when they call the reusable workflow with ``secrets:``. + +For example, ``slow-checks.yml`` keeps its reusable workflow header at the top of the +file: + +.. code-block:: yaml + + name: Slow-Checks + + on: + workflow_call: + secrets: + EXAMPLE_SECRET: + required: true + + .. _workflow_matrix: From b807b23c3a0f370fae11dc98bbaa5e3c47d8698e Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 15:15:17 +0200 Subject: [PATCH 12/30] Add changelog entry --- doc/changes/unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 4fcaebf47..ea818553f 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -12,6 +12,7 @@ so ITDE-related test flows use the configured Exasol baseline and unit-test help ## Feature * #878: Added Nox session `workflow:audit` which uses `zizmor` and added it in `checks.yml` +* #872: Added `custom_workflow_secrets` to `BaseConfig` so that tuples of secrets can be defined for custom workflows, like `slow-checks.yml` ## Refactoring From 7487e6b2c34118476d7c0c8e3f28d03eeda82fed Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 18 Jun 2026 15:15:43 +0200 Subject: [PATCH 13/30] Run format:fix --- test/unit/util/workflows/render_yaml_test.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index a48ff4968..2ed60e596 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -306,9 +306,7 @@ def test_omits_block_when_extension_is_missing( assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_includes_if_block_when_extension_is_present( - test_yml, project_config - ): + def test_includes_if_block_when_extension_is_present(test_yml, project_config): input_yaml = """ jobs: run-unit-tests: @@ -373,9 +371,7 @@ def test_includes_if_block_when_extension_is_present( assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_includes_extension_with_multiple_secrets( - test_yml, project_config - ): + def test_includes_extension_with_multiple_secrets(test_yml, project_config): input_yaml = """ jobs: run-unit-tests: From 7d2e44ac52217e0af7276dec0f6ec58764be42d9 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 19 Jun 2026 09:51:58 +0200 Subject: [PATCH 14/30] Make get_standard_yaml a separate function and extract parse_yaml_text out and use instead --- exasol/toolbox/util/workflows/render_yaml.py | 46 ++++-- test/unit/util/workflows/render_yaml_test.py | 163 +++++++++++++------ 2 files changed, 145 insertions(+), 64 deletions(-) diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index 49a3f2696..3e02db891 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -37,6 +37,30 @@ ) +def get_standard_yaml() -> YAML: + """ + Prepare standard YAML class. + """ + yaml = YAML() + yaml.width = 200 + yaml.preserve_quotes = True + yaml.sort_base_mapping_type_on_output = False # type: ignore + yaml.indent(mapping=2, sequence=4, offset=2) + return yaml + + +def parse_yaml_text(origin_path: Path, workflow_string: str) -> CommentedMap: + """ + Parse YAML from text while keeping the origin path for diagnostics. + """ + try: + yaml = get_standard_yaml() + logger.debug("Parse %s with ruamel-yaml", origin_path) + return yaml.load(workflow_string) + except YAMLError as ex: + raise YamlParsingError(file_path=origin_path) from ex + + @dataclass(frozen=True) class YamlRenderer: """ @@ -49,18 +73,6 @@ class YamlRenderer: github_template_dict: dict[str, Any] file_path: Path - @staticmethod - def _get_standard_yaml() -> YAML: - """ - Prepare standard YAML class. - """ - yaml = YAML() - yaml.width = 200 - yaml.preserve_quotes = True - yaml.sort_base_mapping_type_on_output = False # type: ignore - yaml.indent(mapping=2, sequence=4, offset=2) - return yaml - def _render_with_jinja(self, input_str: str) -> str: """ Render the template with Jinja. @@ -81,19 +93,17 @@ def get_yaml_dict(self) -> CommentedMap: raw_content = self.file_path.read_text() try: workflow_string = self._render_with_jinja(raw_content) - yaml = self._get_standard_yaml() - logger.debug("Parse template with ruamel-yaml") - return yaml.load(workflow_string) except TemplateError as ex: raise TemplateRenderingError(file_path=self.file_path) from ex - except YAMLError as ex: - raise YamlParsingError(file_path=self.file_path) from ex + return parse_yaml_text( + origin_path=self.file_path, workflow_string=workflow_string + ) def get_as_string(self, yaml_dict: CommentedMap) -> str: """ Output a YAML string. """ - yaml = self._get_standard_yaml() + yaml = get_standard_yaml() try: logger.debug("Output workflow as string") with io.StringIO() as stream: diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index 2ed60e596..6e3f9f7f1 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -16,6 +16,7 @@ ) from exasol.toolbox.util.workflows.render_yaml import ( YamlRenderer, + parse_yaml_text, ) @@ -177,7 +178,67 @@ def test_preserves_list_format(test_yml, yaml_renderer): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(input_yaml) @staticmethod - def test_parsing_fails_when_yaml_malformed(test_yml, yaml_renderer): + def test_yaml_cannot_output_to_string(test_yml, yaml_renderer): + input_yaml = """ + steps: + # Comment in nested area + - name: SCM Checkout # Comment inline + uses: actions/checkout@v6 + # Comment in step + """ + content = cleandoc(input_yaml) + test_yml.write_text(content) + + yaml_dict = yaml_renderer.get_yaml_dict() + yaml_dict["steps"][0]["name"] = lambda x: x + 1 + + with pytest.raises(YamlOutputError, match="could not be output") as ex: + yaml_renderer.get_as_string(yaml_dict) + + assert isinstance(ex.value.__cause__, RepresenterError) + assert "cannot represent an object" in str(ex.value.__cause__) + + +class TestParseYamlText: + @staticmethod + def test_parsing_succeeds(tmp_path): + input_yaml = """ + name: Build & Publish + + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + + jobs: + cd-job: + name: Continuous Delivery + permissions: + contents: write + """ + + yaml_dict = parse_yaml_text( + origin_path=tmp_path / "dummy.yml", workflow_string=cleandoc(input_yaml) + ) + + assert yaml_dict == { + "name": "Build & Publish", + "on": { + "workflow_call": { + "secrets": {"PYPI_TOKEN": {"required": True}}, + } + }, + "jobs": { + "cd-job": { + "name": "Continuous Delivery", + "permissions": {"contents": "write"}, + } + }, + } + + @staticmethod + def test_parsing_fails_when_yaml_malformed(tmp_path): bad_template = """ name: Publish Documentation @@ -195,37 +256,18 @@ def test_parsing_fails_when_yaml_malformed(test_yml, yaml_renderer): - name: SCM Checkout uses: actions/checkout@v5 """ - test_yml.write_text(cleandoc(bad_template)) with pytest.raises( YamlParsingError, match="Check for invalid YAML syntax." ) as ex: - yaml_renderer.get_yaml_dict() + parse_yaml_text( + origin_path=tmp_path / "dummy.yml", + workflow_string=cleandoc(bad_template), + ) assert isinstance(ex.value.__cause__, ParserError) assert "while parsing a block collection" in str(ex.value.__cause__) - @staticmethod - def test_yaml_cannot_output_to_string(test_yml, yaml_renderer): - input_yaml = """ - steps: - # Comment in nested area - - name: SCM Checkout # Comment inline - uses: actions/checkout@v6 - # Comment in step - """ - content = cleandoc(input_yaml) - test_yml.write_text(content) - - yaml_dict = yaml_renderer.get_yaml_dict() - yaml_dict["steps"][0]["name"] = lambda x: x + 1 - - with pytest.raises(YamlOutputError, match="could not be output") as ex: - yaml_renderer.get_as_string(yaml_dict) - - assert isinstance(ex.value.__cause__, RepresenterError) - assert "cannot represent an object" in str(ex.value.__cause__) - class TestYamlRendererJinja: @staticmethod @@ -272,9 +314,15 @@ def test_omits_block_when_extension_is_missing( id: check-out-repository uses: actions/checkout@v6 - (% if workflow_extension.fast_tests %) + (% if custom_workflows["fast-tests-extension"].exists %) fast-tests-extension: uses: ./.github/workflows/fast-tests-extension.yml + (% if custom_workflows["fast-tests-extension"].secrets %) + secrets: + (% for secret_name in custom_workflows["fast-tests-extension"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% endif %) @@ -324,9 +372,15 @@ def test_includes_if_block_when_extension_is_present(test_yml, project_config): id: check-out-repository uses: actions/checkout@v6 - (% if workflow_extension.fast_tests %) + (% if custom_workflows["fast-tests-extension"].exists %) fast-tests-extension: uses: ./.github/workflows/fast-tests-extension.yml + (% if custom_workflows["fast-tests-extension"].secrets %) + secrets: + (% for secret_name in custom_workflows["fast-tests-extension"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% endif %) @@ -351,13 +405,27 @@ def test_includes_if_block_when_extension_is_present(test_yml, project_config): fast-tests-extension: uses: ./.github/workflows/fast-tests-extension.yml + secrets: + FAST_TEST_SECRET: ${{ secrets.FAST_TEST_SECRET }} permissions: contents: read """ workflow_directory = project_config.github_workflow_directory workflow_directory.mkdir(parents=True) - (workflow_directory / "fast-tests-extension.yml").touch() + (workflow_directory / "fast-tests-extension.yml").write_text( + cleandoc( + """ + name: Fast-Tests-Extension + + on: + workflow_call: + secrets: + FAST_TEST_SECRET: + required: true + """ + ) + ) content = cleandoc(input_yaml) test_yml.write_text(content) @@ -389,12 +457,12 @@ def test_includes_extension_with_multiple_secrets(test_yml, project_config): id: check-out-repository uses: actions/checkout@v6 - (% if workflow_extension.merge_gate %) + (% if custom_workflows["merge-gate"].exists %) merge-gate-extension: uses: ./.github/workflows/merge-gate-extension.yml - (% if secrets.merge_gate_extension %) + (% if custom_workflows["merge-gate-extension"].secrets %) secrets: - (% for secret_name in secrets.merge_gate_extension %) + (% for secret_name in custom_workflows["merge-gate-extension"].secrets %) (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) @@ -431,27 +499,30 @@ def test_includes_extension_with_multiple_secrets(test_yml, project_config): """ workflow_directory = project_config.github_workflow_directory workflow_directory.mkdir(parents=True) - (workflow_directory / "merge-gate-extension.yml").touch() - - custom_workflow_secrets = project_config.custom_workflow_secrets.model_copy( - update={ - "merge_gate_extension": ( - "MERGE_GATE_SECRET", - "ANOTHER_SECRET", - ) - } - ) - updated_project_config = project_config.model_copy( - update={"custom_workflow_secrets": custom_workflow_secrets} - ) - yaml_renderer = YamlRenderer( - github_template_dict=updated_project_config.github_template_dict, - file_path=test_yml, + (workflow_directory / "merge-gate-extension.yml").write_text( + cleandoc( + """ + name: Merge-Gate-Extension + + on: + workflow_call: + secrets: + MERGE_GATE_SECRET: + required: true + ANOTHER_SECRET: + required: true + """ + ) ) content = cleandoc(input_yaml) test_yml.write_text(content) + yaml_renderer = YamlRenderer( + github_template_dict=project_config.github_template_dict, + file_path=test_yml, + ) + yaml_dict = yaml_renderer.get_yaml_dict() assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) From 56adacb52bba6854928e9f9a2742d0c450e43b00 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 19 Jun 2026 09:53:04 +0200 Subject: [PATCH 15/30] Add custom workflow class which loads a file and can extract secrets from it --- .../toolbox/util/workflows/custom_workflow.py | 48 ++++++++++++++++ exasol/toolbox/util/workflows/exceptions.py | 2 +- .../util/workflows/custom_workflow_test.py | 55 +++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 exasol/toolbox/util/workflows/custom_workflow.py create mode 100644 test/unit/util/workflows/custom_workflow_test.py diff --git a/exasol/toolbox/util/workflows/custom_workflow.py b/exasol/toolbox/util/workflows/custom_workflow.py new file mode 100644 index 000000000..f654f890e --- /dev/null +++ b/exasol/toolbox/util/workflows/custom_workflow.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from pathlib import Path + +from pydantic import ( + BaseModel, + ConfigDict, +) +from ruamel.yaml import ( + CommentedMap, +) + +from exasol.toolbox.util.workflows.render_yaml import parse_yaml_text + + +class CustomWorkflow(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + file_path: Path + yaml_content: CommentedMap + + @classmethod + def load_from_file(cls, file_path: Path) -> CustomWorkflow: + workflow_string = file_path.read_text() + yaml = parse_yaml_text(origin_path=file_path, workflow_string=workflow_string) + return cls(file_path=file_path, yaml_content=yaml) + + def extract_secrets(self) -> tuple[str, ...]: + """Return the secret names declared for ``workflow_call``. + + The reusable workflow must declare them near the top level of the file + like this: + + .. code-block:: yaml + + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + SONAR_TOKEN: + required: true + """ + workflow_call = self.yaml_content.get("on", {}).get("workflow_call") + if isinstance(workflow_call, dict): + if secrets := workflow_call.get("secrets", {}): + return tuple(secrets.keys()) + return () diff --git a/exasol/toolbox/util/workflows/exceptions.py b/exasol/toolbox/util/workflows/exceptions.py index b66acd3e6..1ec4b5479 100644 --- a/exasol/toolbox/util/workflows/exceptions.py +++ b/exasol/toolbox/util/workflows/exceptions.py @@ -31,7 +31,7 @@ class YamlOutputError(YamlError): class YamlParsingError(YamlError): """ - Raised when the rendered template is not a valid YAML file, as it cannot be + Raised when the loaded YAML is not a valid YAML file, as it cannot be parsed by ruamel-yaml. """ diff --git a/test/unit/util/workflows/custom_workflow_test.py b/test/unit/util/workflows/custom_workflow_test.py new file mode 100644 index 000000000..14fc74b53 --- /dev/null +++ b/test/unit/util/workflows/custom_workflow_test.py @@ -0,0 +1,55 @@ +from inspect import cleandoc +from pathlib import Path + +import pytest + +from exasol.toolbox.util.workflows.custom_workflow_extractor import CustomWorkflow + + +@pytest.fixture +def test_yml(tmp_path: Path) -> Path: + return tmp_path / "test.yml" + + +class TestCustomWorkflow: + yaml_with_secrets = cleandoc(""" + name: Build & Publish + + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + SONAR_TOKEN: + required: true + """) + + yaml_without_secrets = cleandoc(""" + name: Build & Publish + + on: + workflow_call: + """) + + def test_load_from_file(self, test_yml): + test_yml.write_text(self.yaml_with_secrets) + + custom_workflow = CustomWorkflow.load_from_file(file_path=test_yml) + + assert custom_workflow.yaml_content + + def test_extract_secrets_when_present(self, test_yml): + test_yml.write_text(self.yaml_with_secrets) + + custom_workflow = CustomWorkflow.load_from_file(file_path=test_yml) + secrets = custom_workflow.extract_secrets() + + assert secrets == ("PYPI_TOKEN", "SONAR_TOKEN") + + def test_extract_secrets_when_no_secrets_present(self, test_yml): + test_yml.write_text(self.yaml_without_secrets) + + custom_workflow = CustomWorkflow.load_from_file(file_path=test_yml) + secrets = custom_workflow.extract_secrets() + + assert secrets == () From 248c34be6610280eb4bec5b6669932410f7e8b58 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 19 Jun 2026 09:57:21 +0200 Subject: [PATCH 16/30] Add docstring to classes to distinguish --- exasol/toolbox/util/workflows/custom_workflow.py | 8 ++++++++ exasol/toolbox/util/workflows/workflow.py | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/exasol/toolbox/util/workflows/custom_workflow.py b/exasol/toolbox/util/workflows/custom_workflow.py index f654f890e..03ff8791a 100644 --- a/exasol/toolbox/util/workflows/custom_workflow.py +++ b/exasol/toolbox/util/workflows/custom_workflow.py @@ -14,6 +14,14 @@ class CustomWorkflow(BaseModel): + """A project-owned workflow used for seeded workflows and extensions. + + These workflows are seeded by the PTB or extend PTB-provided workflows, but + they are maintained by the project itself rather than the PTB. See + `Not Maintained by the PTB `__ + and `Workflow Extensions `__. + """ + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) file_path: Path diff --git a/exasol/toolbox/util/workflows/workflow.py b/exasol/toolbox/util/workflows/workflow.py index 9d96293be..8b6d73199 100644 --- a/exasol/toolbox/util/workflows/workflow.py +++ b/exasol/toolbox/util/workflows/workflow.py @@ -26,6 +26,12 @@ class Workflow(BaseModel): + """A PTB-maintained GitHub workflow rendered from a workflow template. + + These workflows are provided and maintained by the PTB. See + `Maintained by the PTB `__. + """ + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) template_path: Path From d073c03e87f5183f9de4f3ee0aed05f94fb75f80 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 19 Jun 2026 10:16:21 +0200 Subject: [PATCH 17/30] Add custom_workflow_extractor --- .../workflows/custom_workflow_extractor.py | 72 +++++++++ .../custom_workflow_extractor_test.py | 149 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 exasol/toolbox/util/workflows/custom_workflow_extractor.py create mode 100644 test/unit/util/workflows/custom_workflow_extractor_test.py diff --git a/exasol/toolbox/util/workflows/custom_workflow_extractor.py b/exasol/toolbox/util/workflows/custom_workflow_extractor.py new file mode 100644 index 000000000..01ea302f3 --- /dev/null +++ b/exasol/toolbox/util/workflows/custom_workflow_extractor.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from pathlib import Path + +from pydantic import ( + BaseModel, + ConfigDict, +) + +from exasol.toolbox.util.workflows.custom_workflow import CustomWorkflow + + +class CustomWorkflowExtractor(BaseModel): + model_config = ConfigDict(frozen=True) + + github_workflow_directory: Path + sonar_token_name: str + custom_workflows: tuple[str, ...] = ( + "cd-extension", + "fast-tests-extension", + "merge-gate-extension", + "slow-checks", + ) + + def _build_custom_workflow_entry( + self, + workflow: str, + ) -> dict[str, bool | tuple[str, ...]]: + file_path = self.github_workflow_directory / f"{workflow}.yml" + + secrets = () + if file_path.is_file(): + custom_workflow = CustomWorkflow.load_from_file(file_path=file_path) + secrets = custom_workflow.extract_secrets() + + return { + "exists": file_path.exists(), + "secrets": secrets, + } + + def _build_merge_gate_entry( + self, custom_workflows_dict: dict[str, bool | tuple[str, ...]] + ) -> dict[str, bool | tuple[str, ...]]: + return { + "exists": True, + "secrets": custom_workflows_dict["merge-gate-extension"]["secrets"] + + custom_workflows_dict["slow-checks"]["secrets"] + # from the `report.yml` + + (self.sonar_token_name,), + } + + def build_custom_workflow_dict( + self, + ) -> dict[str, dict[str, bool | tuple[str, ...]]]: + """ + Build the template metadata used to specify whether a custom workflow is + present and which secrets its caller must pass through. + + The secret names are extracted from the reusable workflow itself via + :meth:`CustomWorkflow.extract_secrets`. + """ + + custom_workflows_dict = {} + for workflow in self.custom_workflows: + custom_workflows_dict[workflow] = self._build_custom_workflow_entry( + workflow=workflow, + ) + + custom_workflows_dict["merge-gate"] = self._build_merge_gate_entry( + custom_workflows_dict + ) + return custom_workflows_dict diff --git a/test/unit/util/workflows/custom_workflow_extractor_test.py b/test/unit/util/workflows/custom_workflow_extractor_test.py new file mode 100644 index 000000000..53e43e9f1 --- /dev/null +++ b/test/unit/util/workflows/custom_workflow_extractor_test.py @@ -0,0 +1,149 @@ +from inspect import cleandoc + +import pytest + +from exasol.toolbox.util.workflows.custom_workflow_extractor import ( + CustomWorkflowExtractor, +) + + +@pytest.fixture +def workflow_directory(tmp_path): + workflow_directory = tmp_path / ".github" / "workflows" + workflow_directory.mkdir(parents=True) + return workflow_directory + + +@pytest.fixture +def custom_workflow_extractor(workflow_directory): + return CustomWorkflowExtractor( + github_workflow_directory=workflow_directory, + sonar_token_name="SONAR_TOKEN", + ) + + +class TestBuildCustomWorkflowEntry: + @staticmethod + def test_file_does_not_exist(custom_workflow_extractor): + custom_workflow_entry = custom_workflow_extractor._build_custom_workflow_entry( + workflow="slow-checks" + ) + + assert custom_workflow_entry == {"exists": False, "secrets": ()} + + @staticmethod + def test_file_does_not_contain_secrets( + custom_workflow_extractor, workflow_directory + ): + workflow = "slow-checks" + workflow_path = workflow_directory / f"{workflow}.yml" + workflow_path.write_text(cleandoc(""" + name: Slow-Checks + + on: + workflow_call: + """)) + + custom_workflow_entry = custom_workflow_extractor._build_custom_workflow_entry( + workflow=workflow + ) + + assert custom_workflow_entry == {"exists": True, "secrets": ()} + + @staticmethod + def test_file_contains_secret(custom_workflow_extractor, workflow_directory): + workflow = "slow-checks" + workflow_path = workflow_directory / f"{workflow}.yml" + workflow_path.write_text(cleandoc(""" + name: Slow-Checks + + on: + workflow_call: + secrets: + SLOW_CHECK_SECRET: + required: true + """)) + + custom_workflow_entry = custom_workflow_extractor._build_custom_workflow_entry( + workflow=workflow + ) + + assert custom_workflow_entry == { + "exists": True, + "secrets": ("SLOW_CHECK_SECRET",), + } + + +class TestBuildMergeGateEntry: + @staticmethod + def test_build_merge_gate_entry(custom_workflow_extractor): + custom_workflows_dict = { + "merge-gate-extension": {"exists": True, "secrets": ("EXT_SECRET",)}, + "slow-checks": {"exists": True, "secrets": ("SLOW_SECRET",)}, + } + + merge_gate_entry = custom_workflow_extractor._build_merge_gate_entry( + custom_workflows_dict + ) + + assert merge_gate_entry == { + "exists": True, + "secrets": ("EXT_SECRET", "SLOW_SECRET", "SONAR_TOKEN"), + } + + +class TestBuildCustomWorkflowDict: + default_custom_workflow_dict = { + "cd-extension": {"exists": False, "secrets": ()}, + "fast-tests-extension": {"exists": False, "secrets": ()}, + "merge-gate-extension": {"exists": False, "secrets": ()}, + "slow-checks": {"exists": False, "secrets": ()}, + "merge-gate": {"exists": True, "secrets": ("SONAR_TOKEN",)}, + } + + def test_no_custom_workflows_exist(self, custom_workflow_extractor): + custom_workflow_dict = custom_workflow_extractor.build_custom_workflow_dict() + + assert custom_workflow_dict == self.default_custom_workflow_dict + + @pytest.mark.parametrize( + "workflow", + ( + "cd-extension", + "fast-tests-extension", + "merge-gate-extension", + "slow-checks", + ), + ) + def test_custom_workflow_written_to( + self, + custom_workflow_extractor, + workflow_directory, + workflow, + ): + secret = f"{workflow.upper()}_SECRET" + workflow_path = workflow_directory / f"{workflow}.yml" + workflow_path.write_text(cleandoc(f""" + name: {workflow} + + on: + workflow_call: + secrets: + {secret}: + required: true + """)) + + custom_workflow_dict = custom_workflow_extractor.build_custom_workflow_dict() + + expected_custom_workflow_dict = self.default_custom_workflow_dict.copy() + expected_custom_workflow_dict[workflow] = custom_workflow_dict[workflow] = { + "exists": True, + "secrets": (secret,), + } + if workflow in ("merge-gate-extension", "slow-checks"): + expected_custom_workflow_dict["merge-gate"]["secrets"] = ( + secret, + "SONAR_TOKEN", + ) + + assert custom_workflow_dict == expected_custom_workflow_dict From 0f2adbe53cd20d468961c624e62d4fe12d6aa30c Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 22 Jun 2026 09:49:35 +0200 Subject: [PATCH 18/30] Update workflows to use custom_workflows as source --- exasol/toolbox/config.py | 58 ++------------ .../toolbox/templates/github/workflows/cd.yml | 6 +- .../toolbox/templates/github/workflows/ci.yml | 4 +- .../templates/github/workflows/fast-tests.yml | 8 +- .../templates/github/workflows/merge-gate.yml | 16 ++-- .../github/workflows/periodic-validation.yml | 4 +- test/unit/config_test.py | 80 ++----------------- 7 files changed, 37 insertions(+), 139 deletions(-) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 822f1b350..de221b329 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -26,6 +26,9 @@ PLUGIN_ATTR_NAME, ) from exasol.toolbox.util.version import Version +from exasol.toolbox.util.workflows.custom_workflow_extractor import ( + CustomWorkflowExtractor, +) WORKFLOW_HEADER_PREFIX = ( "# Generated and maintained by the exasol-toolbox.\n" @@ -137,35 +140,6 @@ def check_minimum_version(cls, v: str, info: ValidationInfo) -> str: return v -class CustomWorkflowSecrets(BaseModel): - cd_extension: tuple[str, ...] = Field( - default=(), - description=""" - This tuple defines the string names of secrets needed to pass to the - cd-extension.yml. - """, - ) - merge_gate_extension: tuple[str, ...] = Field( - default=(), - description=""" - This tuple defines the string names of secrets needed to pass to the - merge-gate-extension.yml. - """, - ) - slow_checks: tuple[str, ...] = Field( - default=(), - description=""" - This tuple defines the string names of secrets needed to pass to the - slow_checks.yml. - """, - ) - - def get_secrets_dict(self) -> dict[str, tuple[str, ...]]: - secrets = self.model_dump(exclude_computed_fields=True) - secrets["merge_gate"] = self.merge_gate_extension + self.slow_checks - return secrets - - class BaseConfig(BaseModel): """ Basic configuration for projects using the PTB @@ -236,12 +210,6 @@ class BaseConfig(BaseModel): are supported. """, ) - custom_workflow_secrets: CustomWorkflowSecrets = Field( - default=CustomWorkflowSecrets(), - description=""" - This object is used to set the secret arrays for custom workflows. - """, - ) @computed_field # type: ignore[misc] @property @@ -351,31 +319,19 @@ def github_template_dict(self) -> dict[str, Any]: Dictionary of variables to dynamically render Jinja2 templates into valid YAML configurations. """ - cd_extension = self.github_workflow_directory / "cd-extension.yml" - fast_tests_extension = ( - self.github_workflow_directory / "fast-tests-extension.yml" + custom_workflow_extractor = CustomWorkflowExtractor( + github_workflow_directory=self.github_workflow_directory, + sonar_token_name=self.sonar_token_name, ) - merge_gate_extension = ( - self.github_workflow_directory / "merge-gate-extension.yml" - ) - - secrets = self.custom_workflow_secrets.get_secrets_dict() - # merge-gate.yml also calls report.yml and needs Sonar token - secrets["merge_gate"] += (self.sonar_token_name,) return { + "custom_workflows": custom_workflow_extractor.build_custom_workflow_dict(), "dependency_manager_version": self.dependency_manager.version, "minimum_python_version": self.minimum_python_version, "os_version": self.os_version, "python_versions": self.python_versions, - "secrets": secrets, "sonar_token_name": self.sonar_token_name, "workflow_header": f"{WORKFLOW_HEADER_PREFIX}{__version__}.", - "workflow_extension": { - "cd": cd_extension.is_file(), - "fast_tests": fast_tests_extension.is_file(), - "merge_gate": merge_gate_extension.is_file(), - }, } @computed_field # type: ignore[misc] diff --git a/exasol/toolbox/templates/github/workflows/cd.yml b/exasol/toolbox/templates/github/workflows/cd.yml index 45b77545b..ae52b429c 100644 --- a/exasol/toolbox/templates/github/workflows/cd.yml +++ b/exasol/toolbox/templates/github/workflows/cd.yml @@ -24,14 +24,14 @@ jobs: secrets: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - (% if workflow_extension.cd %) + (% if custom_workflows["cd-extension"].exists %) cd-extension: uses: ./.github/workflows/cd-extension.yml needs: - check-release-tag - (% if secrets.cd_extension %) + (% if custom_workflows["cd-extension"].secrets %) secrets: - (% for secret_name in secrets.cd_extension %) + (% for secret_name in custom_workflows["cd-extension"].secrets %) (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/ci.yml b/exasol/toolbox/templates/github/workflows/ci.yml index 34c2424e2..c3c095328 100644 --- a/exasol/toolbox/templates/github/workflows/ci.yml +++ b/exasol/toolbox/templates/github/workflows/ci.yml @@ -9,9 +9,9 @@ jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - (% if secrets.merge_gate %) + (% if custom_workflows["merge-gate"].secrets %) secrets: - (% for secret_name in secrets.merge_gate %) + (% for secret_name in custom_workflows["merge-gate"].secrets %) (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/fast-tests.yml b/exasol/toolbox/templates/github/workflows/fast-tests.yml index 4fe31c32b..ea53e650b 100644 --- a/exasol/toolbox/templates/github/workflows/fast-tests.yml +++ b/exasol/toolbox/templates/github/workflows/fast-tests.yml @@ -42,9 +42,15 @@ jobs: include-hidden-files: true overwrite: false -(% if workflow_extension.fast_tests %) +(% if custom_workflows["fast-tests-extension"].exists %) fast-tests-extension: uses: ./.github/workflows/fast-tests-extension.yml + (% if custom_workflows["fast-tests-extension"].secrets %) + secrets: + (% for secret_name in custom_workflows["fast-tests-extension"].secrets %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index ba08e8046..7b693ac25 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -3,9 +3,9 @@ name: Merge-Gate on: workflow_call: - (% if secrets.merge_gate %) + (% if custom_workflows["merge-gate"].secrets %) secrets: - (% for secret_name in secrets.merge_gate %) + (% for secret_name in custom_workflows["merge-gate"].secrets %) (( secret_name )): required: true (% endfor %) @@ -53,21 +53,21 @@ jobs: needs: - approve-run-slow-tests uses: ./.github/workflows/slow-checks.yml - (% if secrets.slow_checks %) + (% if custom_workflows["slow-checks"].secrets %) secrets: - (% for secret_name in secrets.slow_checks %) + (% for secret_name in custom_workflows["slow-checks"].secrets %) (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) permissions: contents: read - (% if workflow_extension.merge_gate %) + (% if custom_workflows["merge-gate-extension"].exists %) merge-gate-extension: uses: ./.github/workflows/merge-gate-extension.yml - (% if secrets.merge_gate_extension %) + (% if custom_workflows["merge-gate-extension"].secrets %) secrets: - (% for secret_name in secrets.merge_gate_extension %) + (% for secret_name in custom_workflows["merge-gate-extension"].secrets %) (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) @@ -87,7 +87,7 @@ jobs: - run-fast-checks - run-fast-tests - run-slow-checks - (% if workflow_extension.merge_gate %) + (% if custom_workflows["merge-gate-extension"].exists %) - merge-gate-extension (% endif %) # To prevent accidentally merges, this step is required. For more details diff --git a/exasol/toolbox/templates/github/workflows/periodic-validation.yml b/exasol/toolbox/templates/github/workflows/periodic-validation.yml index 99368728f..017f1c0fb 100644 --- a/exasol/toolbox/templates/github/workflows/periodic-validation.yml +++ b/exasol/toolbox/templates/github/workflows/periodic-validation.yml @@ -45,9 +45,9 @@ jobs: uses: ./.github/workflows/slow-checks.yml needs: - restrict-to-default-branch - (% if secrets.slow_checks %) + (% if custom_workflows["slow-checks"].secrets %) secrets: - (% for secret_name in secrets.slow_checks %) + (% for secret_name in custom_workflows["slow-checks"].secrets %) (( secret_name )): ${{ secrets.(( secret_name )) }} (% endfor %) (% endif %) diff --git a/test/unit/config_test.py b/test/unit/config_test.py index 4b7be1c3f..77a370253 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -10,7 +10,6 @@ from exasol.toolbox.config import ( DEFAULT_EXCLUDED_PATHS, BaseConfig, - CustomWorkflowSecrets, DependencyManager, minimum_declared_version, valid_version_string, @@ -43,11 +42,6 @@ def test_works_as_defined(tmp_path, test_project_config_factory): assert config_dump == { "add_to_excluded_python_paths": (), "create_major_version_tags": False, - "custom_workflow_secrets": { - "cd_extension": (), - "merge_gate_extension": (), - "slow_checks": (), - }, "dependency_manager": {"name": "poetry", "version": "2.3.0"}, "documentation_path": root_path / "doc", "exasol_versions": ("8.29.13", "2025.1.8"), @@ -55,10 +49,16 @@ def test_works_as_defined(tmp_path, test_project_config_factory): "github_workflow_directory": tmp_path / ".github" / "workflows", "github_workflow_patcher_yaml": None, "github_template_dict": { + "custom_workflows": { + "cd-extension": {"exists": False, "secrets": ()}, + "fast-tests-extension": {"exists": False, "secrets": ()}, + "merge-gate-extension": {"exists": False, "secrets": ()}, + "slow-checks": {"exists": False, "secrets": ()}, + "merge-gate": {"exists": True, "secrets": ("SONAR_TOKEN",)}, + }, "dependency_manager_version": "2.3.0", "minimum_python_version": "3.10", "os_version": "ubuntu-24.04", - "sonar_token_name": "SONAR_TOKEN", "python_versions": ( "3.10", "3.11", @@ -66,17 +66,7 @@ def test_works_as_defined(tmp_path, test_project_config_factory): "3.13", "3.14", ), - "secrets": { - "cd_extension": (), - "merge_gate": ("SONAR_TOKEN",), - "merge_gate_extension": (), - "slow_checks": (), - }, - "workflow_extension": { - "cd": False, - "fast_tests": False, - "merge_gate": False, - }, + "sonar_token_name": "SONAR_TOKEN", }, "minimum_exasol_version": "8.29.13", "minimum_python_version": "3.10", @@ -144,60 +134,6 @@ def sonar_token_name(self) -> str: return "SONAR_ANOTHER_TOKEN" -class TestCustomWorkflowSecrets: - @staticmethod - def test_get_secrets_dict_defaults(): - custom_workflow_secrets = CustomWorkflowSecrets() - secrets = custom_workflow_secrets.get_secrets_dict() - - assert secrets == { - "cd_extension": (), - "merge_gate": (), - "merge_gate_extension": (), - "slow_checks": (), - } - - @staticmethod - def test_get_secrets_dict_with_cd_extension_override(): - cd_ext_secret = "CD_SECRET" - - custom_workflow_secrets = CustomWorkflowSecrets( - cd_extension=(cd_ext_secret,), - ) - secrets = custom_workflow_secrets.get_secrets_dict() - - assert secrets == { - "cd_extension": (cd_ext_secret,), - "merge_gate": (), - "merge_gate_extension": (), - "slow_checks": (), - } - - @staticmethod - def test_get_secrets_dict_merges_merge_gate_extension_and_slow_checks(): - cd_ext_secret = "CD_SECRET" - merge_gate_ext_secret = "MERGE_GATE_SECRET" - slow_checks_secret = "SLOW_CHECKS_SECRET" - - custom_workflow_secrets = CustomWorkflowSecrets( - cd_extension=(cd_ext_secret,), - merge_gate_extension=(merge_gate_ext_secret, merge_gate_ext_secret), - slow_checks=(slow_checks_secret,), - ) - secrets = custom_workflow_secrets.get_secrets_dict() - - assert secrets == { - "cd_extension": (cd_ext_secret,), - "merge_gate": ( - merge_gate_ext_secret, - merge_gate_ext_secret, - slow_checks_secret, - ), - "merge_gate_extension": (merge_gate_ext_secret, merge_gate_ext_secret), - "slow_checks": (slow_checks_secret,), - } - - def test_expansion_validation_fails_for_invalid_version(): with pytest.raises(ValueError): BaseConfigExpansion(python_versions=("1.f.0",)) From 2be6b73937f9c3c2f499f4083381257e063b4bee Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 22 Jun 2026 11:47:35 +0200 Subject: [PATCH 19/30] Run format:fix --- test/unit/util/workflows/render_yaml_test.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index 6e3f9f7f1..ed8181ddc 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -413,9 +413,7 @@ def test_includes_if_block_when_extension_is_present(test_yml, project_config): """ workflow_directory = project_config.github_workflow_directory workflow_directory.mkdir(parents=True) - (workflow_directory / "fast-tests-extension.yml").write_text( - cleandoc( - """ + (workflow_directory / "fast-tests-extension.yml").write_text(cleandoc(""" name: Fast-Tests-Extension on: @@ -423,9 +421,7 @@ def test_includes_if_block_when_extension_is_present(test_yml, project_config): secrets: FAST_TEST_SECRET: required: true - """ - ) - ) + """)) content = cleandoc(input_yaml) test_yml.write_text(content) @@ -499,9 +495,7 @@ def test_includes_extension_with_multiple_secrets(test_yml, project_config): """ workflow_directory = project_config.github_workflow_directory workflow_directory.mkdir(parents=True) - (workflow_directory / "merge-gate-extension.yml").write_text( - cleandoc( - """ + (workflow_directory / "merge-gate-extension.yml").write_text(cleandoc(""" name: Merge-Gate-Extension on: @@ -511,9 +505,7 @@ def test_includes_extension_with_multiple_secrets(test_yml, project_config): required: true ANOTHER_SECRET: required: true - """ - ) - ) + """)) content = cleandoc(input_yaml) test_yml.write_text(content) From 85198909af4d6b0505344649eea78205ad1a0c84 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 22 Jun 2026 11:53:28 +0200 Subject: [PATCH 20/30] Improve typing --- .../util/workflows/custom_workflow_extractor.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/exasol/toolbox/util/workflows/custom_workflow_extractor.py b/exasol/toolbox/util/workflows/custom_workflow_extractor.py index 01ea302f3..68f094614 100644 --- a/exasol/toolbox/util/workflows/custom_workflow_extractor.py +++ b/exasol/toolbox/util/workflows/custom_workflow_extractor.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +from typing import TypedDict from pydantic import ( BaseModel, @@ -10,6 +11,11 @@ from exasol.toolbox.util.workflows.custom_workflow import CustomWorkflow +class CustomWorkflowEntry(TypedDict): + exists: bool + secrets: tuple[str, ...] + + class CustomWorkflowExtractor(BaseModel): model_config = ConfigDict(frozen=True) @@ -25,10 +31,10 @@ class CustomWorkflowExtractor(BaseModel): def _build_custom_workflow_entry( self, workflow: str, - ) -> dict[str, bool | tuple[str, ...]]: + ) -> CustomWorkflowEntry: file_path = self.github_workflow_directory / f"{workflow}.yml" - secrets = () + secrets: tuple[str, ...] = () if file_path.is_file(): custom_workflow = CustomWorkflow.load_from_file(file_path=file_path) secrets = custom_workflow.extract_secrets() @@ -39,8 +45,8 @@ def _build_custom_workflow_entry( } def _build_merge_gate_entry( - self, custom_workflows_dict: dict[str, bool | tuple[str, ...]] - ) -> dict[str, bool | tuple[str, ...]]: + self, custom_workflows_dict: dict[str, CustomWorkflowEntry] + ) -> CustomWorkflowEntry: return { "exists": True, "secrets": custom_workflows_dict["merge-gate-extension"]["secrets"] @@ -51,7 +57,7 @@ def _build_merge_gate_entry( def build_custom_workflow_dict( self, - ) -> dict[str, dict[str, bool | tuple[str, ...]]]: + ) -> dict[str, CustomWorkflowEntry]: """ Build the template metadata used to specify whether a custom workflow is present and which secrets its caller must pass through. From 6e7519d1e3bb755be8fb4f3ee7b68e3367345902 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 22 Jun 2026 12:45:58 +0200 Subject: [PATCH 21/30] Remove CustomWorkflowSecrets from docs as changed and not part of external API --- doc/api/custom_workflow_secrets.rst | 12 ------------ doc/api/index.rst | 1 - 2 files changed, 13 deletions(-) delete mode 100644 doc/api/custom_workflow_secrets.rst diff --git a/doc/api/custom_workflow_secrets.rst b/doc/api/custom_workflow_secrets.rst deleted file mode 100644 index 5d58289d7..000000000 --- a/doc/api/custom_workflow_secrets.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. _custom_workflow_secrets: - -CustomWorkflowSecrets -===================== - -.. currentmodule:: exasol.toolbox.config - -.. autoclass:: CustomWorkflowSecrets - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource diff --git a/doc/api/index.rst b/doc/api/index.rst index 90e79ce62..8ad43c961 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -7,6 +7,5 @@ :maxdepth: 2 base_config - custom_workflow_secrets workflow_exceptions workflow_patcher_config From d49fbaacb0ca328152148858a6543ea5bfc70b2a Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 22 Jun 2026 12:47:12 +0200 Subject: [PATCH 22/30] Make clear it's a major release due to `all-extras` --- doc/changes/unreleased.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index a0967229f..839933f83 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,7 +2,7 @@ ## Summary -Updated the nox DB-version default to come from `BaseConfig` instead of the hardcoded `7.1.9`, +In this major release, the nox DB-version default was updated to come from `BaseConfig` instead of the hardcoded `7.1.9`, so ITDE-related test flows use the configured Exasol baseline and unit-test help no longer advertises `--db-version`. ## Feature From 565e71aea706b422c617c39d792d99d87b461e10 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 22 Jun 2026 12:53:12 +0200 Subject: [PATCH 23/30] Update unreleased.md to latest implementation --- doc/changes/unreleased.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 839933f83..69708daf6 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -5,6 +5,21 @@ In this major release, the nox DB-version default was updated to come from `BaseConfig` instead of the hardcoded `7.1.9`, so ITDE-related test flows use the configured Exasol baseline and unit-test help no longer advertises `--db-version`. +The `github_template_dict.custom_workflows` entry now auto-detects secret names from +custom workflow files and passes them into PTB-controlled workflow templates. + +For example: + +.. code-block:: yaml + + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + SONAR_TOKEN: + required: true + ## Feature * #874: Added the `security` label to dependency update PR creation @@ -17,7 +32,7 @@ so ITDE-related test flows use the configured Exasol baseline and unit-test help ## Feature * #878: Added Nox session `workflow:audit` which uses `zizmor` and added it in `checks.yml` -* #872: Added `custom_workflow_secrets` to `BaseConfig` so that tuples of secrets can be defined for custom workflows, like `slow-checks.yml` +* #872: Added `custom_workflows` to `github_template_dict` for automatic custom workflow secret extraction ## Refactoring From efd8870fbddca71deb93dfbf84a77c7f9ddbac9e Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 22 Jun 2026 12:57:32 +0200 Subject: [PATCH 24/30] Update unreleased.md so summary is more substantial and accurate --- doc/changes/unreleased.md | 40 ++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 69708daf6..483c9d44c 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,23 +2,28 @@ ## Summary -In this major release, the nox DB-version default was updated to come from `BaseConfig` instead of the hardcoded `7.1.9`, -so ITDE-related test flows use the configured Exasol baseline and unit-test help no longer advertises `--db-version`. - -The `github_template_dict.custom_workflows` entry now auto-detects secret names from -custom workflow files and passes them into PTB-controlled workflow templates. - -For example: - -.. code-block:: yaml - - on: - workflow_call: - secrets: - PYPI_TOKEN: - required: true - SONAR_TOKEN: - required: true +In this major release, several modifications were made to the PTB's workflow templates and actions: + +* the nox DB-version default was updated to come from `BaseConfig` instead of the + hardcoded `7.1.9`, so ITDE-related test flows use the configured Exasol baseline + and unit-test help no longer advertises `--db-version`. +* the `github_template_dict.custom_workflows` entry now auto-detects secret names + from custom workflow files and passes them into PTB-controlled workflow templates. + For example: + + ```yaml + on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + SONAR_TOKEN: + required: true + ``` +* the Python environment GitHub action now accepts `extras` as a comma-separated + list, which makes it easier to pass multiple optional dependency groups in one + value. Additionally, it supports `all-extras`, so that all extras are installed + without further specification needed. ## Feature @@ -37,6 +42,7 @@ For example: ## Refactoring * #744: Extracted shared minimum-version selection logic into `minimum_declared_version()` +* #699: Switched `extras` in the Python environment GitHub action to comma-separation ## Security From ae550f67a980885726fcdbdb4d3d780150d138eb Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 22 Jun 2026 13:01:09 +0200 Subject: [PATCH 25/30] Update workflow_variables.rst to new implementation --- .../github_workflows/workflow_variables.rst | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index f509cc284..d54a6de29 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -22,18 +22,16 @@ standardized baseline that can be overridden in individual projects. Custom Workflow Secrets ^^^^^^^^^^^^^^^^^^^^^^^ -If your project needs to pass secrets into project-controlled workflows, configure -the ``custom_workflow_secrets`` field on :class:`exasol.toolbox.config.BaseConfig`. -That field uses :class:`exasol.toolbox.config.CustomWorkflowSecrets` and lets you -define separate secret tuples for specific workflows. See the API reference for -:class:`exasol.toolbox.config.CustomWorkflowSecrets` for the exact structure. +The PTB extracts secret names from reusable custom workflow files and exposes them +through :py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict` under the +``custom_workflows`` entry. PTB-controlled workflow templates use those extracted +names when they call reusable workflows and forward secrets via ``secrets:``. -Those custom secrets must be declared at the top of the reusable workflow file, under -``on.workflow_call`` and before ``jobs``. The generated workflows rely on that shape -when they call the reusable workflow with ``secrets:``. +To make a custom workflow compatible with this extraction, declare its secrets at the +top of the reusable workflow file under ``on.workflow_call`` and before ``jobs``. +The extractor reads that section and collects the secret names automatically. -For example, ``slow-checks.yml`` keeps its reusable workflow header at the top of the -file: +For example, ``slow-checks.yml`` can define its reusable workflow header like this: .. code-block:: yaml @@ -42,10 +40,13 @@ file: on: workflow_call: secrets: - EXAMPLE_SECRET: + PYPI_TOKEN: + required: true + SONAR_TOKEN: required: true - +Those extracted secret names are then made available to the PTB templates that +reference the custom workflow. .. _workflow_matrix: From c1c867e1306c8909911c5d91934621eb6c51b994 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Mon, 22 Jun 2026 13:06:14 +0200 Subject: [PATCH 26/30] Overhaul documentation to be clearer and link to each other --- .../features/github_workflows/index.rst | 30 +++++++++++++++---- .../github_workflows/workflow_variables.rst | 2 ++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index 35db31df4..f93d879b4 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -36,10 +36,10 @@ compare the rendered workflow templates against the files in ``.github/workflows Workflows --------- -The PTB has three categories of workflows: +The PTB has two categories of workflows: #. those maintained by the PTB, which can be modified using the :ref:`workflow_patcher`. - #. those which are seeded by the PTB but owned and maintained by the project after initial creation. - #. those which extend the PTB-provided workflows and are maintained by the project (not the PTB). + #. custom workflows, which are project-owned and either seeded by the PTB or + extend PTB-provided workflows. Maintained by the PTB ^^^^^^^^^^^^^^^^^^^^^ @@ -108,8 +108,26 @@ Maintained by the PTB the action, and uploads the results to Sonar. -Not Maintained by the PTB -^^^^^^^^^^^^^^^^^^^^^^^^^ +Custom Workflows +^^^^^^^^^^^^^^^^ + +Custom workflows are project-owned. The two custom workflow forms are: + +* workflows seeded once by the PTB and then owned by the project +* workflow extensions that are enabled when the project adds the corresponding file + + +The PTB detects whether the relevant workflow file is present and activates a block +where is it called in corresponding calling workflow. If the file is absent, the PTB +does not add that block to the PTB-controlled workflows. + +For these custom workflows to work correctly, the reusable workflow must follow +the secret declaration format described in :ref:`custom_workflow_secrets` so the +calling PTB workflow can pass the required secrets through. + + +Workflows Initially Seeded by the PTB +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The PTB seeds these workflows for new projects, but after that the project owns them and PTB regeneration does not overwrite them. @@ -126,7 +144,7 @@ them and PTB regeneration does not overwrite them. - Runs long-running checks, which typically involve an Exasol database instance. Workflow Extensions -^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~ To use a workflow extension, a user must simply add the file to their project's ``.github/workflows`` directory. The PTB checks that this file exists, and if it does, diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index d54a6de29..01f0ff384 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -19,6 +19,8 @@ standardized baseline that can be overridden in individual projects. :start-at: github_template_dict :end-before: @computed_field +.. _custom_workflow_secrets: + Custom Workflow Secrets ^^^^^^^^^^^^^^^^^^^^^^^ From 9fe600c7cd5079e38e09ba33318d8ab16616b3b4 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 23 Jun 2026 10:58:35 +0200 Subject: [PATCH 27/30] Apply wording improvements --- doc/changes/unreleased.md | 2 +- doc/user_guide/features/github_workflows/index.rst | 11 ++++++++--- exasol/toolbox/util/workflows/custom_workflow.py | 3 +-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 483c9d44c..bc349a5ba 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -4,7 +4,7 @@ In this major release, several modifications were made to the PTB's workflow templates and actions: -* the nox DB-version default was updated to come from `BaseConfig` instead of the +* the default DB-version was updated to come from `BaseConfig` instead of the hardcoded `7.1.9`, so ITDE-related test flows use the configured Exasol baseline and unit-test help no longer advertises `--db-version`. * the `github_template_dict.custom_workflows` entry now auto-detects secret names diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index f93d879b4..5bbc15e94 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -36,10 +36,15 @@ compare the rendered workflow templates against the files in ``.github/workflows Workflows --------- -The PTB has two categories of workflows: +The PTB manages two categories of workflows: #. those maintained by the PTB, which can be modified using the :ref:`workflow_patcher`. - #. custom workflows, which are project-owned and either seeded by the PTB or - extend PTB-provided workflows. + #. custom workflows, which are project-owned. + +Custom workflows can optionally be +* seeded by the PTB, i.e. PTB generates an initial version but ignores future changes. +* extend PTB-provided workflows, i.e. ending in `-extension.yml` + +Besides that you can also create individual additional workflow files with are ignored by the PTB. Maintained by the PTB ^^^^^^^^^^^^^^^^^^^^^ diff --git a/exasol/toolbox/util/workflows/custom_workflow.py b/exasol/toolbox/util/workflows/custom_workflow.py index 03ff8791a..49c375b61 100644 --- a/exasol/toolbox/util/workflows/custom_workflow.py +++ b/exasol/toolbox/util/workflows/custom_workflow.py @@ -18,8 +18,7 @@ class CustomWorkflow(BaseModel): These workflows are seeded by the PTB or extend PTB-provided workflows, but they are maintained by the project itself rather than the PTB. See - `Not Maintained by the PTB `__ - and `Workflow Extensions `__. + `Custom Workflows `__. """ model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) From 8983952cb3211a4372f0ebffa11a396b6067f1ae Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 23 Jun 2026 11:05:06 +0200 Subject: [PATCH 28/30] Fix wording. We really don't 'manage' custom workflows --- doc/user_guide/features/github_workflows/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index 5bbc15e94..9b3e85b9b 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -36,7 +36,7 @@ compare the rendered workflow templates against the files in ``.github/workflows Workflows --------- -The PTB manages two categories of workflows: +The PTB allows for two categories of workflows: #. those maintained by the PTB, which can be modified using the :ref:`workflow_patcher`. #. custom workflows, which are project-owned. @@ -44,7 +44,7 @@ Custom workflows can optionally be * seeded by the PTB, i.e. PTB generates an initial version but ignores future changes. * extend PTB-provided workflows, i.e. ending in `-extension.yml` -Besides that you can also create individual additional workflow files with are ignored by the PTB. +Besides that, you can also create individual workflow files which are ignored by the PTB. Maintained by the PTB ^^^^^^^^^^^^^^^^^^^^^ From 7139d39d03aafe853d0372249cba1c43427047a0 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 23 Jun 2026 11:08:11 +0200 Subject: [PATCH 29/30] Update example to make sense --- doc/user_guide/troubleshooting/handle_zizmor_findings.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/user_guide/troubleshooting/handle_zizmor_findings.rst b/doc/user_guide/troubleshooting/handle_zizmor_findings.rst index 4a3efb191..13bcba6c8 100644 --- a/doc/user_guide/troubleshooting/handle_zizmor_findings.rst +++ b/doc/user_guide/troubleshooting/handle_zizmor_findings.rst @@ -32,7 +32,11 @@ A typical line-level ignore looks like this: .. code-block:: yaml - secrets: inherit # zizmor: ignore[github-env] - This shared action is used by many workflows, and downstream steps need `poetry` on PATH; we do not have a safer replacement yet. + - name: Set up Poetry (${{ inputs.poetry-version }}) + shell: bash + run: | # zizmor: ignore[github-env] - This shared action is used by many workflows, and downstream steps need `poetry` on PATH; we do not have a safer replacement yet. + POETRY_VERSION="${INPUTS_POETRY_VERSION}" "$PYTHON_BINARY" "${{ github.action_path }}/ext/get_poetry.py" + echo "$HOME/.local/bin" >> $GITHUB_PATH Use configuration rules in ``.zizmor.yml`` only when the finding is genuinely project-wide. If you add a temporary rule while working through a batch of From c3b8521430b700908f327d773454f40ae5096032 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 23 Jun 2026 11:13:57 +0200 Subject: [PATCH 30/30] Add one more entry to summary --- doc/changes/unreleased.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index bc349a5ba..1697863df 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -24,6 +24,14 @@ In this major release, several modifications were made to the PTB's workflow tem list, which makes it easier to pass multiple optional dependency groups in one value. Additionally, it supports `all-extras`, so that all extras are installed without further specification needed. +* the new `workflow:audit` Nox session runs `zizmor` against GitHub Actions and + reusable workflows, so security checks are part of the normal `checks.yml` + pipeline instead of being a separate manual step. It also keeps the audit + configuration in the project root via `.zizmor.yml`; see the + [zizmor configuration guide](https://exasol.github.io/python-toolbox/main/user_guide/features/managing_dependencies/zizmor_configuration.html) + and the + [troubleshooting guide for findings](https://exasol.github.io/python-toolbox/main/user_guide/troubleshooting/handle_zizmor_findings.html) + for details on tuning or suppressing findings locally. ## Feature