diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 5898b29..4ba1b65 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 - name: Build Artifacts run: poetry build diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..11673c9 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,24 @@ +name: CD + +on: + push: + tags: + - '**' + +jobs: + + check-tag-version-job: + name: Check Release Tag + uses: ./.github/workflows/check-release-tag.yml + + cd-job: + name: Continuous Delivery + uses: ./.github/workflows/build-and-publish.yml + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + + publish-docs: + needs: [ cd-job ] + name: Publish Documentation + uses: ./.github/workflows/gh-pages.yml + diff --git a/.github/workflows/check-release-tag.yml b/.github/workflows/check-release-tag.yml index 256e58f..5415fd0 100644 --- a/.github/workflows/check-release-tag.yml +++ b/.github/workflows/check-release-tag.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 - name: Check Tag Version # make sure the pushed/created tag matched the project version diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index caa8c4e..f2d9e31 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,11 +1,15 @@ name: Checks -on: workflow_call +on: + workflow_call: + secrets: + ALTERNATIVE_GITHUB_TOKEN: + required: false jobs: - version-check-job: - name: Version Check & Build Documentation + Version-Check: + name: Version runs-on: ubuntu-latest steps: @@ -15,41 +19,138 @@ jobs: fetch-depth: 0 - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 - name: Check Version(s) - run: poetry run version-check exasol/bucketfs/version.py + run: poetry run version-check `poetry run python -c "from noxconfig import PROJECT_CONFIG; print(PROJECT_CONFIG.version_file)"` + echo "Please enable the version check by replacing this output with shell command bellow:" + echo "" + echo "poetry run version-check <>" + echo "" + echo "Note: <> needs to point to the version file of the project (version.py)." + exit 1 + + Documentation: + name: Docs + needs: [ Version-Check ] + runs-on: ubuntu-latest + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 - name: Build Documentation run: | poetry run python -m nox -s docs:build - lint-job: - name: Linting and Type checks (Python-${{ matrix.python-version }}) - needs: [version-check-job] + Lint: + name: Linting (Python-${{ matrix.python-version }}) + needs: [ Version-Check ] runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] steps: - name: SCM Checkout uses: actions/checkout@v4 - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 with: python-version: ${{ matrix.python-version }} - - name: Run Tests + - name: Run lint run: poetry run nox -s lint:code + - name: Upload Artifacts + uses: actions/upload-artifact@v4.4.0 + with: + name: lint-python${{ matrix.python-version }} + path: .lint.txt + include-hidden-files: true + + Type-Check: + name: Type Checking (Python-${{ matrix.python-version }}) + needs: [ Version-Check ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.9", "3.10", "3.11", "3.12" ] + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 + with: + python-version: ${{ matrix.python-version }} + - name: Run type-check run: poetry run nox -s lint:typing + Security: + name: Security Checks (Python-${{ matrix.python-version }}) + needs: [ Version-Check ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.9", "3.10", "3.11", "3.12" ] + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Run security linter + run: poetry run nox -s lint:security + - name: Upload Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: - name: .lint.txt - path: .lint.txt + name: security-python${{ matrix.python-version }} + path: .security.json + include-hidden-files: true + + Tests: + name: Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + needs: [ Documentation, Lint, Type-Check, Security ] + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.9", "3.10", "3.11", "3.12" ] + exasol-version: ["7.1.9"] + + steps: + - name: SCM Checkout + uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Run Tests and Compute Coverage + run: poetry run nox -s test:coverage -- -- --db-version ${{ matrix.exasol-version }} + + - name: Upload Artifacts + uses: actions/upload-artifact@v4.4.0 + with: + name: coverage-python${{ matrix.python-version }} + path: .coverage + include-hidden-files: true + diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml deleted file mode 100644 index d7d9383..0000000 --- a/.github/workflows/ci-cd.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: CI/CD - -on: - push: - tags: - - '**' - -jobs: - - check-tag-version-job: - name: Check Release Tag - uses: ./.github/workflows/check-release-tag.yml - - ci-job: - name: Checks - needs: [ check-tag-version-job ] - uses: ./.github/workflows/checks.yml - - tests-job: - name: Tests (Python-${{ matrix.python-version }}) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] - - steps: - - uses: actions/checkout@v4 - - - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 - with: - python-version: ${{ matrix.python-version }} - poetry-version: 1.2.2 - - - name: Run Legacy Tests - run: poetry run pytest --backend=onprem test_legacy - - cd-job: - name: Continuous Delivery - needs: [ ci-job ] - uses: ./.github/workflows/build-and-publish.yml - secrets: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33bbfe2..44d85e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,92 +2,28 @@ name: CI on: pull_request: + +# why would we need push / pull_request? +# I prefer just pull_request + +# push: +# branches-ignore: +# - "github-pages/*" +# - "gh-pages/*" +# - "main" +# - "master" +# pull_request: +# types: [opened, reopened] schedule: # “At 00:00 on every 7th day-of-month from 1 through 31.” (https://crontab.guru) - cron: "0 0 1/7 * *" jobs: - ci-job: - name: Checks - uses: ./.github/workflows/checks.yml - - fast-tests: - name: Tests (Python-${{ matrix.python-version }}) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] - - steps: - - uses: actions/checkout@v4 - - - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 - with: - python-version: ${{ matrix.python-version }} - poetry-version: 1.2.2 + CI: + uses: ./.github/workflows/merge-gate.yml + secrets: inherit - - name: Run Unit Tests - run: poetry run nox -s test:unit - - - name: Upload Artifacts - uses: actions/upload-artifact@v3 - with: - name: .coverage - path: .coverage - - metrics: - needs: [ fast-tests ] + Metrics: + needs: [ CI ] uses: ./.github/workflows/report.yml - - gate-1: - name: Gate 1 - Regular CI - needs: [ fast-tests ] - runs-on: ubuntu-latest - steps: - - name: Branch Protection - run: true - - slow-test-detection: - name: Run Slow or Expensive Tests (e.g. SaaS)? - runs-on: ubuntu-latest - steps: - - name: Detect Slow Tests - run: true - environment: - slow-tests - - run-slow-tests: - name: Run Slow or Expensive Tests (e.g. SaaS) if Requested - runs-on: ubuntu-latest - needs: [ slow-test-detection ] - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] - - steps: - - uses: actions/checkout@v4 - - - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 - with: - python-version: ${{ matrix.python-version }} - poetry-version: "1.2.2" - - - name: Run Integration Tests - env: - SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} - SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} - SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} - run: poetry run pytest --backend=all test/integration - - gate-2: - name: Gate 2 - Allow Merge - runs-on: ubuntu-latest - needs: [ run-slow-tests ] - steps: - - name: Branch Protection - run: true diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 23bd753..dfdac63 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -1,6 +1,8 @@ name: Publish Documentation -on: workflow_call +on: + workflow_call: + workflow_dispatch: jobs: @@ -10,13 +12,15 @@ jobs: steps: - name: SCM Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 - name: Build Documentation run: | - poetry run python -m nox -s build-docs + poetry run nox -s docs:multiversion - name: Deploy uses: JamesIves/github-pages-deploy-action@v4.6.0 diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml new file mode 100644 index 0000000..7888e07 --- /dev/null +++ b/.github/workflows/merge-gate.yml @@ -0,0 +1,38 @@ +name: Merge-Gate + +on: + workflow_call: + secrets: + ALTERNATIVE_GITHUB_TOKEN: + required: false + +jobs: + + fast-checks: + name: Fast + uses: ./.github/workflows/checks.yml + + slow-checks: + name: Slow + runs-on: ubuntu-latest + environment: slow-tests # manual-approval + steps: + - name: Tests + env: + SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} + SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} + SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} + run: poetry run pytest --backend=all test/integration + + # This job ensures inputs have been executed successfully. + approve-merge: + name: Allow Merge + runs-on: ubuntu-latest + # If you need additional jobs to be part of the merge gate, add them below + needs: [ fast-checks, slow-checks ] + + # Each job requires a step, so we added this dummy step. + steps: + - name: Approve + run: | + echo "Merge Approved" diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 8ed5fb7..bb74336 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -9,7 +9,6 @@ on: jobs: report: - name: Generate Status Report runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} @@ -21,35 +20,35 @@ jobs: fetch-depth: 0 - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 + uses: exasol/python-toolbox/.github/actions/python-environment@0.18.0 - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: path: ./artifacts - name: Copy Artifacts into Root Folder - if: ${{ hashFiles('./artifacts') != '' }} working-directory: ./artifacts run: | - cp .coverage/.coverage ../ - cp .lint.txt/.lint.txt ../ + cp coverage-python3.9/.coverage ../ + cp lint-python3.9/.lint.txt ../ + cp security-python3.9/.security.json ../ - name: Generate Report - run: poetry run nox -s report -- -- --format json | tee metrics.json + run: poetry run nox -s project:report -- -- --format json | tee metrics.json - name: Upload Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v4.4.0 with: name: metrics.json path: metrics.json - name: Generate GitHub Summary - if: ${{ hashFiles('./artifacts') != '' }} run: | echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY - poetry run nox -s report -- -- --format markdown >> $GITHUB_STEP_SUMMARY + poetry run nox -s project:report -- -- --format markdown >> $GITHUB_STEP_SUMMARY echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY echo -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY cat .lint.txt >> $GITHUB_STEP_SUMMARY + poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY