diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index efafb012c..1d81326d6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -40,14 +40,17 @@ jobs: run: | poetry run python -m nox -s docs:build + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-python.yml + Lint: name: Linting (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -70,12 +73,11 @@ jobs: Type-Check: name: Type Checking (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -91,12 +93,11 @@ jobs: Security: name: Security Checks (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -134,16 +135,14 @@ jobs: run: poetry run nox -s project:format Tests: - name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) - needs: [ Documentation, Lint, Type-Check, Security, Format] + name: Unit-Tests (Python-${{ matrix.python-version }}) + needs: [ Documentation, Lint, Type-Check, Security, Format, build-matrix ] runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] - exasol-version: [ "7.1.9" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -155,7 +154,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run Tests and Collect Coverage - run: poetry run nox -s test:unit -- -- --coverage --db-version ${{ matrix.exasol-version }} + run: poetry run nox -s test:unit -- -- --coverage - name: Upload Artifacts uses: actions/upload-artifact@v4.4.0 diff --git a/.github/workflows/matrix-all.yml b/.github/workflows/matrix-all.yml new file mode 100644 index 000000000..4d7c60b6d --- /dev/null +++ b/.github/workflows/matrix-all.yml @@ -0,0 +1,30 @@ +name: Build Matrix (All Versions) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the all versions build matrix" + value: ${{ jobs.all_versions.outputs.matrix }} + +jobs: + all_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + + - name: Generate matrix + run: poetry run nox -s matrix:all + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:all)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/matrix-exasol.yml b/.github/workflows/matrix-exasol.yml new file mode 100644 index 000000000..4a16983cc --- /dev/null +++ b/.github/workflows/matrix-exasol.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Exasol) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the exasol version build matrix" + value: ${{ jobs.exasol_versions.outputs.matrix }} + +jobs: + exasol_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + + - name: Generate matrix + run: poetry run nox -s matrix:exasol + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:exasol)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/matrix-python.yml b/.github/workflows/matrix-python.yml new file mode 100644 index 000000000..f12f746f7 --- /dev/null +++ b/.github/workflows/matrix-python.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Python) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the python version build matrix" + value: ${{ jobs.python_versions.outputs.matrix }} + +jobs: + python_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + + - name: Generate matrix + run: poetry run nox -s matrix:python + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:python)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 14d2fe0ef..89c49f7ab 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -8,8 +8,13 @@ on: jobs: + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-all.yml + Tests: name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + needs: [ build-matrix ] runs-on: ubuntu-latest # Even though the environment "manual-approval" will be created automatically, # it still needs to be configured to require interactive review. @@ -19,9 +24,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] - exasol-version: [ "7.1.9" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index fb13787e3..11136f408 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1,5 +1,42 @@ # Unreleased +## 🚨 Breaking Changes +* **Matrices in CI/CD workflows will be generated automatically now** + + Make sure you have installed all the latest workflow files, especially the newly added ones: + + - `matrix-all.yml` + - `matrix-python.yml` + - `matrix-exasol.yml` + + +## ✨ Added +* Added support for dynamically generated workflow matrices. + + This feature allows you to easily change the test matrices in one place: `noxconfig.py`. + + Note: As usual, there are different ways a user can adjust or change the behavior. In the case of the build matrices, there are three obvious ways: + + - Set the appropriate fields in the `noxconfig.py` project configuration (`PROJECT_CONFIG`): + * `python_versions = [ ... ]` + * `exasol_versions = [ ... ]` + - Overwrite the nox tasks: + * `matrix:all` + * `matrix:python` + * `matrix:exasol` + - Overwrite/replace the matrix generation workflows: + * `matrix-all.yml` + * `matrix-python.yml` + * `matrix-exasol.yml` + + Among all of the above, the safest way is to set the matrix-related fields in your project config object in `noxconfig.py`. + + +## 📚 Documentation + +* Added new entries to the frequently asked questions regarding `multiversion documentation` + + ## 🐞 Fixed * Fixed `index.rst` documentation template to provide the correct underlining length of the main heading diff --git a/doc/faq.rst b/doc/faq.rst index 2363a80ce..77c4c3782 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -23,3 +23,43 @@ Duplicated label error when building documentation -------------------------------------------------- Similar error to :code:`Warning, treated as error: integration-test-docker-environment/doc/changes/changes_0.10.0.md:5:duplicate label summary, other instance in integration-test-docker-environment/doc/changes/changes_0.1.0.md'`, might be caused by sphinx extension `sphinx.ext.autosectionlabel`. Try to remove this extension in `doc/conf.py`. + + +.. _faq_multiversion_build_warnings: + +Warning while building multiversion documentation +-------------------------------------------------- +When running ``nox -s docs:multiversion``, I receive the following warnings during the build: + +.. code-block:: + + WARNING: unknown config value 'smv_metadata_path' in override, ignoring + WARNING: unknown config value 'smv_current_version' in override, ignoring + +If you receive the warnings above, it is very likely that the multiversion extension is not configured in your Sphinx configuration (``conf.py``). Try adding it to your configuration and rerun the build. + +.. code-block:: python + + extensions = [ + ..., + ..., + "exasol.toolbox.sphinx.multiversion", + ] + + +.. _faq_multiversion_selection_missing: + +Missing Version Selection Box in Multiversion Documentation +------------------------------------------------------------ + +I have run ``nox -s docs:multiversion``, but I still do not see any version selection box in the upper right corner before the GitHub symbol. + +This is likely due to :ref:`faq_multiversion_build_warnings` + + +.. _faq_multiversion_limited_versions: + +Limited Previous Versions in Multiversion Documentation +------------------------------------------------------- + +If not all previous versions of the project are available via the version selection box of the multiversion documentation, it is likely due to the fact that the unavailable documentation for those versions was not in a compatible format (there hasn't been a compatible setup of a Sphinx-based documentation). diff --git a/exasol/toolbox/nox/_ci.py b/exasol/toolbox/nox/_ci.py new file mode 100644 index 000000000..603634165 --- /dev/null +++ b/exasol/toolbox/nox/_ci.py @@ -0,0 +1,59 @@ +import json +import logging + +import nox +from nox import Session + +from noxconfig import ( + PROJECT_CONFIG, + Config, +) + +_log = logging.getLogger(__name__) + +_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] +_EXASOL_VERSIONS = ["7.1.9"] + + +def _python_matrix(config: Config): + attr = "python_versions" + python_versions = getattr(config, attr, _PYTHON_VERSIONS) + if not hasattr(config, attr): + _log.warning( + "Config does not contain '%s' setting. Using default: %s", + attr, + _PYTHON_VERSIONS, + ) + return {"python-version": python_versions} + + +def _exasol_matrix(config: Config): + attr = "exasol_versions" + exasol_versions = getattr(config, attr, _EXASOL_VERSIONS) + if not hasattr(config, attr): + _log.warning( + "Config does not contain '%s' setting. Using default: %s", + attr, + _EXASOL_VERSIONS, + ) + return {"exasol-version": exasol_versions} + + +@nox.session(name="matrix:python", python=False) +def python_matrix(session: Session) -> None: + """Output the build matrix for Python versions as JSON.""" + print(json.dumps(_python_matrix(PROJECT_CONFIG))) + + +@nox.session(name="matrix:exasol", python=False) +def exasol_matrix(session: Session) -> None: + """Output the build matrix for Exasol versions as JSON.""" + print(json.dumps(_exasol_matrix(PROJECT_CONFIG))) + + +@nox.session(name="matrix:all", python=False) +def full_matrix(session: Session) -> None: + """Output the full build matrix for Python & Exasol versions as JSON.""" + matrix = _python_matrix(PROJECT_CONFIG) + matrix.update(_exasol_matrix(PROJECT_CONFIG)) + print(json.dumps(matrix)) diff --git a/exasol/toolbox/nox/tasks.py b/exasol/toolbox/nox/tasks.py index 3592abbd7..99fe598df 100644 --- a/exasol/toolbox/nox/tasks.py +++ b/exasol/toolbox/nox/tasks.py @@ -59,7 +59,6 @@ def check(session: Session) -> None: clean_docs, open_docs, ) -from exasol.toolbox.nox._release import prepare_release from exasol.toolbox.nox._shared import ( Mode, _context, @@ -67,6 +66,13 @@ def check(session: Session) -> None: python_files, ) +from exasol.toolbox.nox._ci import ( + python_matrix, + exasol_matrix, + full_matrix, +) + +from exasol.toolbox.nox._release import prepare_release # isort: on # fmt: on diff --git a/exasol/toolbox/templates/github/workflows/checks.yml b/exasol/toolbox/templates/github/workflows/checks.yml index 04896a795..0d737e643 100644 --- a/exasol/toolbox/templates/github/workflows/checks.yml +++ b/exasol/toolbox/templates/github/workflows/checks.yml @@ -46,14 +46,17 @@ jobs: run: | poetry run python -m nox -s docs:build + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-python.yml + Lint: name: Linting (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -76,12 +79,11 @@ jobs: Type-Check: name: Type Checking (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -97,12 +99,11 @@ jobs: Security: name: Security Checks (Python-${{ matrix.python-version }}) - needs: [ Version-Check ] + needs: [ Version-Check, build-matrix ] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -124,7 +125,7 @@ jobs: include-hidden-files: true Format: - name: Format Check (Python-${{ matrix.python-version }}) + name: Format Check runs-on: ubuntu-latest steps: @@ -135,20 +136,19 @@ jobs: uses: exasol/python-toolbox/.github/actions/python-environment@0.19.0 with: python-version: "3.9" + - name: Run format check run: poetry run nox -s project:format Tests: name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) - needs: [ Documentation, Lint, Type-Check, Security, Format ] + needs: [ Documentation, Lint, Type-Check, Security, Format, build-matrix ] runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] - exasol-version: ["7.1.9"] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout @@ -160,7 +160,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run Tests and Collect Coverage - run: poetry run nox -s test:unit -- -- --coverage --db-version ${{ matrix.exasol-version }} + run: poetry run nox -s test:unit -- -- --coverage - name: Upload Artifacts uses: actions/upload-artifact@v4.4.0 diff --git a/exasol/toolbox/templates/github/workflows/matrix-all.yml b/exasol/toolbox/templates/github/workflows/matrix-all.yml new file mode 100644 index 000000000..b23406b89 --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/matrix-all.yml @@ -0,0 +1,30 @@ +name: Build Matrix (All Versions) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the all versions build matrix" + value: ${{ jobs.all_versions.outputs.matrix }} + +jobs: + all_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.19.0 + + - name: Generate matrix + run: poetry run nox -s matrix:all + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:all)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/matrix-exasol.yml b/exasol/toolbox/templates/github/workflows/matrix-exasol.yml new file mode 100644 index 000000000..279b5ae30 --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/matrix-exasol.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Exasol) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the exasol version build matrix" + value: ${{ jobs.exasol_versions.outputs.matrix }} + +jobs: + exasol_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.19.0 + + - name: Generate matrix + run: poetry run nox -s matrix:exasol + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:exasol)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/matrix-python.yml b/exasol/toolbox/templates/github/workflows/matrix-python.yml new file mode 100644 index 000000000..0ee1ee889 --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/matrix-python.yml @@ -0,0 +1,30 @@ +name: Build Matrix (Python) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the python version build matrix" + value: ${{ jobs.python_versions.outputs.matrix }} + +jobs: + python_versions: + + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.19.0 + + - name: Generate matrix + run: poetry run nox -s matrix:python + + - id: set-matrix + run: | + echo "matrix=$(poetry run nox -s matrix:python)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/slow-checks.yml b/exasol/toolbox/templates/github/workflows/slow-checks.yml index 21d635921..b898e9ceb 100644 --- a/exasol/toolbox/templates/github/workflows/slow-checks.yml +++ b/exasol/toolbox/templates/github/workflows/slow-checks.yml @@ -8,8 +8,13 @@ on: jobs: + build-matrix: + name: Generate Build Matrix + uses: ./.github/workflows/matrix-all.yml + Tests: name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + needs: [ build-matrix ] runs-on: ubuntu-latest # Even though the environment "manual-approval" will be created automatically, # it still needs to be configured to require interactive review. @@ -19,9 +24,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} strategy: fail-fast: false - matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] - exasol-version: [ "7.1.9" ] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - name: SCM Checkout diff --git a/noxconfig.py b/noxconfig.py index 59272931b..a570b1ee6 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -45,6 +45,8 @@ class Config: "project-template", "idioms", ) + python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"] + exasol_versions = ["7.1.9"] plugins = [UpdateTemplates] diff --git a/test/integration/cli/workflow-install.t b/test/integration/cli/workflow-install.t index 0ff4992d0..b09b6468d 100644 --- a/test/integration/cli/workflow-install.t +++ b/test/integration/cli/workflow-install.t @@ -11,6 +11,9 @@ Check if all workflows have been installed checks.yml ci.yml gh-pages.yml + matrix-all.yml + matrix-exasol.yml + matrix-python.yml merge-gate.yml pr-merge.yml report.yml diff --git a/test/integration/cli/workflow-list.t b/test/integration/cli/workflow-list.t index c066272f0..cf366104c 100644 --- a/test/integration/cli/workflow-list.t +++ b/test/integration/cli/workflow-list.t @@ -7,6 +7,9 @@ List all available workflows checks ci gh-pages + matrix-all + matrix-exasol + matrix-python merge-gate pr-merge report diff --git a/test/unit/template_test.py b/test/unit/template_test.py index 97e699c71..52d949766 100644 --- a/test/unit/template_test.py +++ b/test/unit/template_test.py @@ -15,6 +15,9 @@ def test_retrieve_workflow_templates(): "checks": "checks.yml", "ci": "ci.yml", "gh-pages": "gh-pages.yml", + "matrix-all": "matrix-all.yml", + "matrix-exasol": "matrix-exasol.yml", + "matrix-python": "matrix-python.yml", "merge-gate": "merge-gate.yml", "pr-merge": "pr-merge.yml", "report": "report.yml", @@ -52,6 +55,9 @@ def test_retrieve_issue_templates(): "checks": "checks.yml", "ci": "ci.yml", "gh-pages": "gh-pages.yml", + "matrix-all": "matrix-all.yml", + "matrix-exasol": "matrix-exasol.yml", + "matrix-python": "matrix-python.yml", "merge-gate": "merge-gate.yml", "pr-merge": "pr-merge.yml", "report": "report.yml", @@ -104,6 +110,9 @@ def test_retrieve_templates(subpackage, expected): "checks.yml", "ci.yml", "gh-pages.yml", + "matrix-all.yml", + "matrix-exasol.yml", + "matrix-python.yml", "merge-gate.yml", "pr-merge.yml", "report.yml",