diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e80c736c0..0a34a268b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,8 @@ jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} permissions: contents: read diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 39fb7ee9d..b77cc4bab 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -4,6 +4,9 @@ name: Merge-Gate on: workflow_call: + secrets: + SONAR_TOKEN: + required: true jobs: run-fast-checks: @@ -47,13 +50,11 @@ jobs: needs: - approve-run-slow-tests uses: ./.github/workflows/slow-checks.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. permissions: contents: read merge-gate-extension: uses: ./.github/workflows/merge-gate-extension.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. permissions: contents: read diff --git a/.github/workflows/periodic-validation.yml b/.github/workflows/periodic-validation.yml index 01b25f5b3..62834750f 100644 --- a/.github/workflows/periodic-validation.yml +++ b/.github/workflows/periodic-validation.yml @@ -46,7 +46,6 @@ jobs: uses: ./.github/workflows/slow-checks.yml needs: - restrict-to-default-branch - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. permissions: contents: read diff --git a/doc/api/custom_workflow_secrets.rst b/doc/api/custom_workflow_secrets.rst new file mode 100644 index 000000000..5d58289d7 --- /dev/null +++ b/doc/api/custom_workflow_secrets.rst @@ -0,0 +1,12 @@ +.. _custom_workflow_secrets: + +CustomWorkflowSecrets +===================== + +.. currentmodule:: exasol.toolbox.config + +.. autoclass:: CustomWorkflowSecrets + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource diff --git a/doc/api/index.rst b/doc/api/index.rst index 8ad43c961..90e79ce62 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -7,5 +7,6 @@ :maxdepth: 2 base_config + custom_workflow_secrets workflow_exceptions workflow_patcher_config diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 4fcaebf47..ea818553f 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -12,6 +12,7 @@ so ITDE-related test flows use the configured Exasol baseline and unit-test help ## Feature * #878: Added Nox session `workflow:audit` which uses `zizmor` and added it in `checks.yml` +* #872: Added `custom_workflow_secrets` to `BaseConfig` so that tuples of secrets can be defined for custom workflows, like `slow-checks.yml` ## Refactoring diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst index b1ee9ba5b..f509cc284 100644 --- a/doc/user_guide/features/github_workflows/workflow_variables.rst +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -19,6 +19,33 @@ standardized baseline that can be overridden in individual projects. :start-at: github_template_dict :end-before: @computed_field +Custom Workflow Secrets +^^^^^^^^^^^^^^^^^^^^^^^ + +If your project needs to pass secrets into project-controlled workflows, configure +the ``custom_workflow_secrets`` field on :class:`exasol.toolbox.config.BaseConfig`. +That field uses :class:`exasol.toolbox.config.CustomWorkflowSecrets` and lets you +define separate secret tuples for specific workflows. See the API reference for +:class:`exasol.toolbox.config.CustomWorkflowSecrets` for the exact structure. + +Those custom secrets must be declared at the top of the reusable workflow file, under +``on.workflow_call`` and before ``jobs``. The generated workflows rely on that shape +when they call the reusable workflow with ``secrets:``. + +For example, ``slow-checks.yml`` keeps its reusable workflow header at the top of the +file: + +.. code-block:: yaml + + name: Slow-Checks + + on: + workflow_call: + secrets: + EXAMPLE_SECRET: + required: true + + .. _workflow_matrix: diff --git a/doc/user_guide/troubleshooting/handle_zizmor_findings.rst b/doc/user_guide/troubleshooting/handle_zizmor_findings.rst index 69916e4fe..4a3efb191 100644 --- a/doc/user_guide/troubleshooting/handle_zizmor_findings.rst +++ b/doc/user_guide/troubleshooting/handle_zizmor_findings.rst @@ -32,7 +32,7 @@ A typical line-level ignore looks like this: .. code-block:: yaml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet. + secrets: inherit # zizmor: ignore[github-env] - This shared action is used by many workflows, and downstream steps need `poetry` on PATH; we do not have a safer replacement yet. Use configuration rules in ``.zizmor.yml`` only when the finding is genuinely project-wide. If you add a temporary rule while working through a batch of diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 4e7211342..822f1b350 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -137,6 +137,35 @@ def check_minimum_version(cls, v: str, info: ValidationInfo) -> str: return v +class CustomWorkflowSecrets(BaseModel): + cd_extension: tuple[str, ...] = Field( + default=(), + description=""" + This tuple defines the string names of secrets needed to pass to the + cd-extension.yml. + """, + ) + merge_gate_extension: tuple[str, ...] = Field( + default=(), + description=""" + This tuple defines the string names of secrets needed to pass to the + merge-gate-extension.yml. + """, + ) + slow_checks: tuple[str, ...] = Field( + default=(), + description=""" + This tuple defines the string names of secrets needed to pass to the + slow_checks.yml. + """, + ) + + def get_secrets_dict(self) -> dict[str, tuple[str, ...]]: + secrets = self.model_dump(exclude_computed_fields=True) + secrets["merge_gate"] = self.merge_gate_extension + self.slow_checks + return secrets + + class BaseConfig(BaseModel): """ Basic configuration for projects using the PTB @@ -207,6 +236,12 @@ class BaseConfig(BaseModel): are supported. """, ) + custom_workflow_secrets: CustomWorkflowSecrets = Field( + default=CustomWorkflowSecrets(), + description=""" + This object is used to set the secret arrays for custom workflows. + """, + ) @computed_field # type: ignore[misc] @property @@ -324,11 +359,16 @@ def github_template_dict(self) -> dict[str, Any]: self.github_workflow_directory / "merge-gate-extension.yml" ) + secrets = self.custom_workflow_secrets.get_secrets_dict() + # merge-gate.yml also calls report.yml and needs Sonar token + secrets["merge_gate"] += (self.sonar_token_name,) + return { "dependency_manager_version": self.dependency_manager.version, "minimum_python_version": self.minimum_python_version, "os_version": self.os_version, "python_versions": self.python_versions, + "secrets": secrets, "sonar_token_name": self.sonar_token_name, "workflow_header": f"{WORKFLOW_HEADER_PREFIX}{__version__}.", "workflow_extension": { diff --git a/exasol/toolbox/templates/github/workflows/cd.yml b/exasol/toolbox/templates/github/workflows/cd.yml index 618063c3e..45b77545b 100644 --- a/exasol/toolbox/templates/github/workflows/cd.yml +++ b/exasol/toolbox/templates/github/workflows/cd.yml @@ -29,7 +29,12 @@ jobs: uses: ./.github/workflows/cd-extension.yml needs: - check-release-tag - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.cd_extension %) + secrets: + (% for secret_name in secrets.cd_extension %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: write (% endif %) diff --git a/exasol/toolbox/templates/github/workflows/ci.yml b/exasol/toolbox/templates/github/workflows/ci.yml index c79cb52d6..34c2424e2 100644 --- a/exasol/toolbox/templates/github/workflows/ci.yml +++ b/exasol/toolbox/templates/github/workflows/ci.yml @@ -9,7 +9,12 @@ jobs: merge-gate: name: Merge Gate uses: ./.github/workflows/merge-gate.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.merge_gate %) + secrets: + (% for secret_name in secrets.merge_gate %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml index a7498dd92..ba08e8046 100644 --- a/exasol/toolbox/templates/github/workflows/merge-gate.yml +++ b/exasol/toolbox/templates/github/workflows/merge-gate.yml @@ -3,6 +3,13 @@ name: Merge-Gate on: workflow_call: + (% if secrets.merge_gate %) + secrets: + (% for secret_name in secrets.merge_gate %) + (( secret_name )): + required: true + (% endfor %) + (% endif %) jobs: run-fast-checks: @@ -46,14 +53,24 @@ jobs: needs: - approve-run-slow-tests uses: ./.github/workflows/slow-checks.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.slow_checks %) + secrets: + (% for secret_name in secrets.slow_checks %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% if workflow_extension.merge_gate %) merge-gate-extension: uses: ./.github/workflows/merge-gate-extension.yml - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.merge_gate_extension %) + secrets: + (% for secret_name in secrets.merge_gate_extension %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read (% endif %) @@ -70,9 +87,9 @@ jobs: - run-fast-checks - run-fast-tests - run-slow-checks - (% if workflow_extension.merge_gate %) + (% if workflow_extension.merge_gate %) - merge-gate-extension - (% endif %) + (% endif %) # To prevent accidentally merges, this step is required. For more details # see: https://github.com/exasol/python-toolbox/issues/563 steps: diff --git a/exasol/toolbox/templates/github/workflows/periodic-validation.yml b/exasol/toolbox/templates/github/workflows/periodic-validation.yml index 34f78370c..99368728f 100644 --- a/exasol/toolbox/templates/github/workflows/periodic-validation.yml +++ b/exasol/toolbox/templates/github/workflows/periodic-validation.yml @@ -45,7 +45,12 @@ jobs: uses: ./.github/workflows/slow-checks.yml needs: - restrict-to-default-branch - secrets: inherit # zizmor: ignore[secrets-inherit] - PTB cannot customize inherited secrets here yet; tracked in https://github.com/exasol/python-toolbox/issues/872. + (% if secrets.slow_checks %) + secrets: + (% for secret_name in secrets.slow_checks %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) permissions: contents: read diff --git a/exasol/toolbox/templates/github/workflows/slow-checks.yml b/exasol/toolbox/templates/github/workflows/slow-checks.yml index b15394bfa..487268d1b 100644 --- a/exasol/toolbox/templates/github/workflows/slow-checks.yml +++ b/exasol/toolbox/templates/github/workflows/slow-checks.yml @@ -21,8 +21,6 @@ jobs: runs-on: "(( os_version ))" permissions: contents: read - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} strategy: fail-fast: false matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} diff --git a/test/unit/config_test.py b/test/unit/config_test.py index 3115d0632..4b7be1c3f 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -10,6 +10,7 @@ from exasol.toolbox.config import ( DEFAULT_EXCLUDED_PATHS, BaseConfig, + CustomWorkflowSecrets, DependencyManager, minimum_declared_version, valid_version_string, @@ -42,6 +43,11 @@ def test_works_as_defined(tmp_path, test_project_config_factory): assert config_dump == { "add_to_excluded_python_paths": (), "create_major_version_tags": False, + "custom_workflow_secrets": { + "cd_extension": (), + "merge_gate_extension": (), + "slow_checks": (), + }, "dependency_manager": {"name": "poetry", "version": "2.3.0"}, "documentation_path": root_path / "doc", "exasol_versions": ("8.29.13", "2025.1.8"), @@ -60,6 +66,12 @@ def test_works_as_defined(tmp_path, test_project_config_factory): "3.13", "3.14", ), + "secrets": { + "cd_extension": (), + "merge_gate": ("SONAR_TOKEN",), + "merge_gate_extension": (), + "slow_checks": (), + }, "workflow_extension": { "cd": False, "fast_tests": False, @@ -132,6 +144,60 @@ def sonar_token_name(self) -> str: return "SONAR_ANOTHER_TOKEN" +class TestCustomWorkflowSecrets: + @staticmethod + def test_get_secrets_dict_defaults(): + custom_workflow_secrets = CustomWorkflowSecrets() + secrets = custom_workflow_secrets.get_secrets_dict() + + assert secrets == { + "cd_extension": (), + "merge_gate": (), + "merge_gate_extension": (), + "slow_checks": (), + } + + @staticmethod + def test_get_secrets_dict_with_cd_extension_override(): + cd_ext_secret = "CD_SECRET" + + custom_workflow_secrets = CustomWorkflowSecrets( + cd_extension=(cd_ext_secret,), + ) + secrets = custom_workflow_secrets.get_secrets_dict() + + assert secrets == { + "cd_extension": (cd_ext_secret,), + "merge_gate": (), + "merge_gate_extension": (), + "slow_checks": (), + } + + @staticmethod + def test_get_secrets_dict_merges_merge_gate_extension_and_slow_checks(): + cd_ext_secret = "CD_SECRET" + merge_gate_ext_secret = "MERGE_GATE_SECRET" + slow_checks_secret = "SLOW_CHECKS_SECRET" + + custom_workflow_secrets = CustomWorkflowSecrets( + cd_extension=(cd_ext_secret,), + merge_gate_extension=(merge_gate_ext_secret, merge_gate_ext_secret), + slow_checks=(slow_checks_secret,), + ) + secrets = custom_workflow_secrets.get_secrets_dict() + + assert secrets == { + "cd_extension": (cd_ext_secret,), + "merge_gate": ( + merge_gate_ext_secret, + merge_gate_ext_secret, + slow_checks_secret, + ), + "merge_gate_extension": (merge_gate_ext_secret, merge_gate_ext_secret), + "slow_checks": (slow_checks_secret,), + } + + def test_expansion_validation_fails_for_invalid_version(): with pytest.raises(ValueError): BaseConfigExpansion(python_versions=("1.f.0",)) diff --git a/test/unit/util/workflows/render_yaml_test.py b/test/unit/util/workflows/render_yaml_test.py index eed3331ff..2ed60e596 100644 --- a/test/unit/util/workflows/render_yaml_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -252,7 +252,9 @@ def test_updates_jinja_variables(test_yml, yaml_renderer): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_does_not_add_jinja_block(test_yml, yaml_renderer, project_config): + def test_omits_block_when_extension_is_missing( + test_yml, yaml_renderer, project_config + ): input_yaml = """ jobs: run-unit-tests: @@ -304,7 +306,7 @@ def test_does_not_add_jinja_block(test_yml, yaml_renderer, project_config): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_adds_jinja_block(test_yml, project_config): + def test_includes_if_block_when_extension_is_present(test_yml, project_config): input_yaml = """ jobs: run-unit-tests: @@ -368,6 +370,91 @@ def test_adds_jinja_block(test_yml, project_config): assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod + def test_includes_extension_with_multiple_secrets(test_yml, project_config): + input_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "(( os_version ))" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: (( python_versions | tojson )) + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + (% if workflow_extension.merge_gate %) + merge-gate-extension: + uses: ./.github/workflows/merge-gate-extension.yml + (% if secrets.merge_gate_extension %) + secrets: + (% for secret_name in secrets.merge_gate_extension %) + (( secret_name )): ${{ secrets.(( secret_name )) }} + (% endfor %) + (% endif %) + permissions: + contents: read + (% endif %) + + """ + expected_yaml = """ + jobs: + run-unit-tests: + name: Unit Tests (Python-${{ matrix.python-versions }}) + runs-on: "ubuntu-24.04" + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + merge-gate-extension: + uses: ./.github/workflows/merge-gate-extension.yml + secrets: + MERGE_GATE_SECRET: ${{ secrets.MERGE_GATE_SECRET }} + ANOTHER_SECRET: ${{ secrets.ANOTHER_SECRET }} + permissions: + contents: read + + """ + workflow_directory = project_config.github_workflow_directory + workflow_directory.mkdir(parents=True) + (workflow_directory / "merge-gate-extension.yml").touch() + + custom_workflow_secrets = project_config.custom_workflow_secrets.model_copy( + update={ + "merge_gate_extension": ( + "MERGE_GATE_SECRET", + "ANOTHER_SECRET", + ) + } + ) + updated_project_config = project_config.model_copy( + update={"custom_workflow_secrets": custom_workflow_secrets} + ) + yaml_renderer = YamlRenderer( + github_template_dict=updated_project_config.github_template_dict, + file_path=test_yml, + ) + + content = cleandoc(input_yaml) + test_yml.write_text(content) + + yaml_dict = yaml_renderer.get_yaml_dict() + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) + @staticmethod def test_jinja_variable_unknown(test_yml, yaml_renderer): input_yaml = """