diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index efafb012c..1d81326d6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -40,14 +40,17 @@ jobs: run: | poetry run python -m nox -s docs:build + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-python.yml + Lint: name: Linting (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -70,12 +73,11 @@ jobs: Type-Check: name: Type Checking (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -91,12 +93,11 @@ jobs: Security: name: Security Checks (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -134,16 +135,14 @@ jobs: run: poetry run nox -s project:format Tests: - name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) - needs: [ Documentation, Lint, Type-Check, Security, Format] + name: Unit-Tests (Python-${{ matrix.python-version }}) + needs: [ Documentation, Lint, Type-Check, Security, Format, build-matrix ] runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] - exasol-version: [ "7.1.9" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -155,7 +154,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run Tests and Collect Coverage - run: poetry run nox -s test:unit -- -- --coverage --db-version ${{ matrix.exasol-version }} + run: poetry run nox -s test:unit -- -- --coverage - name: Upload Artifacts uses: actions/upload-artifact@v4.4.0 diff --git a/.github/workflows/matrix-all.yml b/.github/workflows/matrix-all.yml new file mode 100644 index 000000000..4d7c60b6d --- /dev/null +++ b/.github/workflows/matrix-all.yml @@ -0,0 +1,30 @@ +name: Build Matrix (All Versions) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the all versions build matrix" + value: ${{ jobs.all_versions.outputs.matrix }} + +jobs: + all_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + + - name: Generate matrix + run: poetry run nox -s matrix:all + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:all)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/matrix-exasol.yml b/.github/workflows/matrix-exasol.yml new file mode 100644 index 000000000..4a16983cc --- /dev/null +++ b/.github/workflows/matrix-exasol.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Exasol) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the exasol version build matrix" + value: ${{ jobs.exasol_versions.outputs.matrix }} + +jobs: + exasol_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + + - name: Generate matrix + run: poetry run nox -s matrix:exasol + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:exasol)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/matrix-python.yml b/.github/workflows/matrix-python.yml new file mode 100644 index 000000000..f12f746f7 --- /dev/null +++ b/.github/workflows/matrix-python.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Python) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the python version build matrix" + value: ${{ jobs.python_versions.outputs.matrix }} + +jobs: + python_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + + - name: Generate matrix + run: poetry run nox -s matrix:python + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:python)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 14d2fe0ef..89c49f7ab 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -8,8 +8,13 @@ on: jobs: + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-all.yml + Tests: name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + needs: [ build-matrix ] runs-on: ubuntu-latest # Even though the environment "manual-approval" will be created automatically, # it still needs to be configured to require interactive review. @@ -19,9 +24,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] - exasol-version: [ "7.1.9" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 176ee9244..7b46e8365 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1,9 +1,42 @@ # Unreleased +## 🚨 Breaking Changes +* **Matrices in CI/CD workflows will be generated automatically now** + + Make sure you have installed all the latest workflow files, especially the newly added ones: + + - `matrix-all.yml` + - `matrix-python.yml` + - `matrix-exasol.yml` + + +## ✨ Added +* Added support for dynamically generated workflow matrices. + + This feature allows you to easily change the test matrices in one place: `noxconfig.py`. + + Note: As usual, there are different ways a user can adjust or change the behavior. In the case of the build matrices, there are three obvious ways: + + - Set the appropriate fields in the `noxconfig.py` project configuration (`PROJECT_CONFIG`): + * `python_versions = [ ... ]` + * `exasol_versions = [ ... ]` + - Overwrite the nox tasks: + * `matrix:all` + * `matrix:python` + * `matrix:exasol` + - Overwrite/replace the matrix generation workflows: + * `matrix-all.yml` + * `matrix-python.yml` + * `matrix-exasol.yml` + + Among all of the above, the safest way is to set the matrix-related fields in your project config object in `noxconfig.py`. + + ## 📚 Documentation * Added new entries to the frequently asked questions regarding `multiversion documentation` + ## 🐞 Fixed * Added multi-version extension to Sphinx configuration of the project template diff --git a/exasol/toolbox/nox/_ci.py b/exasol/toolbox/nox/_ci.py new file mode 100644 index 000000000..603634165 --- /dev/null +++ b/exasol/toolbox/nox/_ci.py @@ -0,0 +1,59 @@ +import json +import logging + +import nox +from nox import Session + +from noxconfig import ( + PROJECT_CONFIG, + Config, +) + +_log = logging.getLogger(__name__) + +_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] +_EXASOL_VERSIONS = ["7.1.9"] + + +def _python_matrix(config: Config): + attr = "python_versions" + python_versions = getattr(config, attr, _PYTHON_VERSIONS) + if not hasattr(config, attr): + _log.warning( + "Config does not contain '%s' setting. Using default: %s", + attr, + _PYTHON_VERSIONS, + ) + return {"python-version": python_versions} + + +def _exasol_matrix(config: Config): + attr = "exasol_versions" + exasol_versions = getattr(config, attr, _EXASOL_VERSIONS) + if not hasattr(config, attr): + _log.warning( + "Config does not contain '%s' setting. Using default: %s", + attr, + _EXASOL_VERSIONS, + ) + return {"exasol-version": exasol_versions} + + +@nox.session(name="matrix:python", python=False) +def python_matrix(session: Session) -> None: + """Output the build matrix for Python versions as JSON.""" + print(json.dumps(_python_matrix(PROJECT_CONFIG))) + + +@nox.session(name="matrix:exasol", python=False) +def exasol_matrix(session: Session) -> None: + """Output the build matrix for Exasol versions as JSON.""" + print(json.dumps(_exasol_matrix(PROJECT_CONFIG))) + + +@nox.session(name="matrix:all", python=False) +def full_matrix(session: Session) -> None: + """Output the full build matrix for Python & Exasol versions as JSON.""" + matrix = _python_matrix(PROJECT_CONFIG) + matrix.update(_exasol_matrix(PROJECT_CONFIG)) + print(json.dumps(matrix)) diff --git a/exasol/toolbox/nox/tasks.py b/exasol/toolbox/nox/tasks.py index 3592abbd7..99fe598df 100644 --- a/exasol/toolbox/nox/tasks.py +++ b/exasol/toolbox/nox/tasks.py @@ -59,7 +59,6 @@ def check(session: Session) -> None: clean_docs, open_docs, ) -from exasol.toolbox.nox._release import prepare_release from exasol.toolbox.nox._shared import ( Mode, _context, @@ -67,6 +66,13 @@ def check(session: Session) -> None: python_files, ) +from exasol.toolbox.nox._ci import ( + python_matrix, + exasol_matrix, + full_matrix, +) + +from exasol.toolbox.nox._release import prepare_release # isort: on # fmt: on diff --git a/exasol/toolbox/templates/github/workflows/checks.yml b/exasol/toolbox/templates/github/workflows/checks.yml index 04896a795..0d737e643 100644 --- a/exasol/toolbox/templates/github/workflows/checks.yml +++ b/exasol/toolbox/templates/github/workflows/checks.yml @@ -46,14 +46,17 @@ jobs: run: | poetry run python -m nox -s docs:build + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-python.yml + Lint: name: Linting (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -76,12 +79,11 @@ jobs: Type-Check: name: Type Checking (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -97,12 +99,11 @@ jobs: Security: name: Security Checks (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -124,7 +125,7 @@ jobs: include-hidden-files: true Format: - name: Format Check (Python-${{ matrix.python-version }}) + name: Format Check runs-on: ubuntu-latest steps: @@ -135,20 +136,19 @@ jobs: uses: exasol/python-toolbox/.github/actions/python-environment@0.19.0 with: python-version: "3.9" + - name: Run format check run: poetry run nox -s project:format Tests: name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) - needs: [ Documentation, Lint, Type-Check, Security, Format ] + needs: [ Documentation, Lint, Type-Check, Security, Format, build-matrix ] runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] - exasol-version: ["7.1.9"] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -160,7 +160,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run Tests and Collect Coverage - run: poetry run nox -s test:unit -- -- --coverage --db-version ${{ matrix.exasol-version }} + run: poetry run nox -s test:unit -- -- --coverage - name: Upload Artifacts uses: actions/upload-artifact@v4.4.0 diff --git a/exasol/toolbox/templates/github/workflows/matrix-all.yml b/exasol/toolbox/templates/github/workflows/matrix-all.yml new file mode 100644 index 000000000..b23406b89 --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/matrix-all.yml @@ -0,0 +1,30 @@ +name: Build Matrix (All Versions) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the all versions build matrix" + value: ${{ jobs.all_versions.outputs.matrix }} + +jobs: + all_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.19.0 + + - name: Generate matrix + run: poetry run nox -s matrix:all + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:all)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/matrix-exasol.yml b/exasol/toolbox/templates/github/workflows/matrix-exasol.yml new file mode 100644 index 000000000..279b5ae30 --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/matrix-exasol.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Exasol) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the exasol version build matrix" + value: ${{ jobs.exasol_versions.outputs.matrix }} + +jobs: + exasol_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.19.0 + + - name: Generate matrix + run: poetry run nox -s matrix:exasol + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:exasol)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/matrix-python.yml b/exasol/toolbox/templates/github/workflows/matrix-python.yml new file mode 100644 index 000000000..0ee1ee889 --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/matrix-python.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Python) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the python version build matrix" + value: ${{ jobs.python_versions.outputs.matrix }} + +jobs: + python_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.19.0 + + - name: Generate matrix + run: poetry run nox -s matrix:python + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:python)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/slow-checks.yml b/exasol/toolbox/templates/github/workflows/slow-checks.yml index 21d635921..b898e9ceb 100644 --- a/exasol/toolbox/templates/github/workflows/slow-checks.yml +++ b/exasol/toolbox/templates/github/workflows/slow-checks.yml @@ -8,8 +8,13 @@ on: jobs: + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-all.yml + Tests: name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + needs: [ build-matrix ] runs-on: ubuntu-latest # Even though the environment "manual-approval" will be created automatically, # it still needs to be configured to require interactive review. @@ -19,9 +24,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] - exasol-version: [ "7.1.9" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout diff --git a/noxconfig.py b/noxconfig.py index 59272931b..a570b1ee6 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -45,6 +45,8 @@ class Config: "project-template", "idioms", ) + python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"] + exasol_versions = ["7.1.9"] plugins = [UpdateTemplates] diff --git a/test/integration/cli/workflow-install.t b/test/integration/cli/workflow-install.t index 0ff4992d0..b09b6468d 100644 --- a/test/integration/cli/workflow-install.t +++ b/test/integration/cli/workflow-install.t @@ -11,6 +11,9 @@ Check if all workflows have been installed checks.yml ci.yml gh-pages.yml + matrix-all.yml + matrix-exasol.yml + matrix-python.yml merge-gate.yml pr-merge.yml report.yml diff --git a/test/integration/cli/workflow-list.t b/test/integration/cli/workflow-list.t index c066272f0..cf366104c 100644 --- a/test/integration/cli/workflow-list.t +++ b/test/integration/cli/workflow-list.t @@ -7,6 +7,9 @@ List all available workflows checks ci gh-pages + matrix-all + matrix-exasol + matrix-python merge-gate pr-merge report diff --git a/test/unit/template_test.py b/test/unit/template_test.py index 97e699c71..52d949766 100644 --- a/test/unit/template_test.py +++ b/test/unit/template_test.py @@ -15,6 +15,9 @@ def test_retrieve_workflow_templates(): "checks": "checks.yml", "ci": "ci.yml", "gh-pages": "gh-pages.yml", + "matrix-all": "matrix-all.yml", + "matrix-exasol": "matrix-exasol.yml", + "matrix-python": "matrix-python.yml", "merge-gate": "merge-gate.yml", "pr-merge": "pr-merge.yml", "report": "report.yml", @@ -52,6 +55,9 @@ def test_retrieve_issue_templates(): "checks": "checks.yml", "ci": "ci.yml", "gh-pages": "gh-pages.yml", + "matrix-all": "matrix-all.yml", + "matrix-exasol": "matrix-exasol.yml", + "matrix-python": "matrix-python.yml", "merge-gate": "merge-gate.yml", "pr-merge": "pr-merge.yml", "report": "report.yml", @@ -104,6 +110,9 @@ def test_retrieve_templates(subpackage, expected): "checks.yml", "ci.yml", "gh-pages.yml", + "matrix-all.yml", + "matrix-exasol.yml", + "matrix-python.yml", "merge-gate.yml", "pr-merge.yml", "report.yml",