From 8efc02f2f831f51ffb0a2079dd837b9da7f652b9 Mon Sep 17 00:00:00 2001 From: pgoslatara Date: Mon, 9 Sep 2024 08:42:50 +0200 Subject: [PATCH 1/4] Accounting for Windows runners in post merge/release workflows --- .github/workflows/merge_pipeline.yml | 6 +++++- .github/workflows/post_release_pipeline.yml | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/merge_pipeline.yml b/.github/workflows/merge_pipeline.yml index 24c325cf..4ee7fc3b 100644 --- a/.github/workflows/merge_pipeline.yml +++ b/.github/workflows/merge_pipeline.yml @@ -123,7 +123,7 @@ - name: Run `dbt-bouncer` run: python ./dist/dbt-bouncer.pex --config-file dbt-bouncer-example.yml - pypi-tests: + pip-tests: runs-on: ${{ matrix.operating-system }} strategy: fail-fast: false @@ -144,9 +144,13 @@ - name: Determine python version id: python-version run: | + if [[ ${{ matrix.operating-system }} == "windows-latest" ]]; then + echo "PYTHON_VERSION=$(cat .python-version)" >> $ENV:GITHUB_OUTPUT + else export PYTHON_VERSION=$(cat .python-version) echo "PYTHON_VERSION: $PYTHON_VERSION" echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_OUTPUT + fi - name: Setup Python uses: ./.github/actions/setup_python_env diff --git a/.github/workflows/post_release_pipeline.yml b/.github/workflows/post_release_pipeline.yml index 97311275..e45e6a74 100644 --- a/.github/workflows/post_release_pipeline.yml +++ b/.github/workflows/post_release_pipeline.yml @@ -99,9 +99,13 @@ jobs: - name: Determine python version id: python-version run: | + if [[ ${{ matrix.operating-system }} == "windows-latest" ]]; then + echo "PYTHON_VERSION=$(cat .python-version)" >> $ENV:GITHUB_OUTPUT + else export PYTHON_VERSION=$(cat .python-version) echo "PYTHON_VERSION: $PYTHON_VERSION" echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_OUTPUT + fi - name: Setup Python uses: ./.github/actions/setup_python_env @@ -110,6 +114,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install `dbt-bouncer` + if: matrix.operating-system == 'windows-latest' + run: pip install dbt-bouncer=='("${{ github.ref }}" -Split "/")[2]' + + - name: Install `dbt-bouncer` + if: matrix.operating-system != 'windows-latest' run: pip install dbt-bouncer==$(echo "${{ github.ref }}" | cut -d "/" -f3) - name: Run `dbt-bouncer` From 73238ffcca3dc7c99dd76e8982631d179f17491d Mon Sep 17 00:00:00 2001 From: pgoslatara Date: Mon, 9 Sep 2024 18:33:16 +0200 Subject: [PATCH 2/4] Don't run PIPESTATUS command on Windows --- .github/workflows/ci_pipeline.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml index c1562263..f9545f0f 100644 --- a/.github/workflows/ci_pipeline.yml +++ b/.github/workflows/ci_pipeline.yml @@ -84,6 +84,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run pytest (unit tests) + if: matrix.operating-system == 'windows-latest' + run: make test-unit + + - name: Run pytest (unit tests) + if: matrix.operating-system != 'windows-latest' run: make test-unit | tee pytest-coverage.txt && exit ${PIPESTATUS[0]} - name: Pytest coverage comment From ec23d4f3a2e9c1ae0c2ceb163221ea1e125cb8e6 Mon Sep 17 00:00:00 2001 From: pgoslatara Date: Mon, 9 Sep 2024 20:00:15 +0200 Subject: [PATCH 3/4] More pipeline fixes for Windows --- .github/workflows/ci_pipeline.yml | 21 ++-- .github/workflows/merge_pipeline.yml | 25 ++--- .github/workflows/post_release_pipeline.yml | 25 ++--- makefile | 111 ++++++++++---------- 4 files changed, 95 insertions(+), 87 deletions(-) diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml index f9545f0f..1a2cbba7 100644 --- a/.github/workflows/ci_pipeline.yml +++ b/.github/workflows/ci_pipeline.yml @@ -83,16 +83,16 @@ jobs: poetry-version: ${{ env.POETRY_VERSION }} python-version: ${{ matrix.python-version }} - - name: Run pytest (unit tests) + - name: Run pytest (Windows) if: matrix.operating-system == 'windows-latest' - run: make test-unit + run: make test-windows - name: Run pytest (unit tests) if: matrix.operating-system != 'windows-latest' run: make test-unit | tee pytest-coverage.txt && exit ${PIPESTATUS[0]} - name: Pytest coverage comment - if: matrix.python-version == '3.11' && matrix.operating-system == 'ubuntu-latest' + if: matrix.python-version == '3.11' && matrix.operating-system != 'windows-latest' uses: MishaKav/pytest-coverage-comment@main with: pytest-coverage-path: ./pytest-coverage.txt @@ -101,16 +101,25 @@ jobs: junitxml-path: ./coverage.xml - name: Run pytest (integration tests) + if: matrix.operating-system != 'windows-latest' run: make test-integration - - name: Regenerate dbt artifacts + - name: Regenerate dbt artifacts and run `dbt-bouncer` + if: matrix.operating-system == 'windows-latest' run: | + $env:Path += ';C:\Users\runneradmin\.local\bin' poetry run dbt deps poetry run dbt build poetry run dbt docs generate + poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml - - name: Run `dbt-bouncer` on generated artifacts - run: poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml + - name: Regenerate dbt artifacts and run `dbt-bouncer` + if: matrix.operating-system != 'windows-latest' + run: | + poetry run dbt deps + poetry run dbt build + poetry run dbt docs generate + poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml e2e-tests: needs: [pre-commit] diff --git a/.github/workflows/merge_pipeline.yml b/.github/workflows/merge_pipeline.yml index 4ee7fc3b..d9d0677b 100644 --- a/.github/workflows/merge_pipeline.yml +++ b/.github/workflows/merge_pipeline.yml @@ -141,17 +141,6 @@ - name: Checkout uses: actions/checkout@v4 - - name: Determine python version - id: python-version - run: | - if [[ ${{ matrix.operating-system }} == "windows-latest" ]]; then - echo "PYTHON_VERSION=$(cat .python-version)" >> $ENV:GITHUB_OUTPUT - else - export PYTHON_VERSION=$(cat .python-version) - echo "PYTHON_VERSION: $PYTHON_VERSION" - echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_OUTPUT - fi - - name: Setup Python uses: ./.github/actions/setup_python_env with: @@ -164,11 +153,19 @@ - name: Run `dbt-bouncer` run: dbt-bouncer --config-file dbt-bouncer-example.yml - - name: Regenerate dbt artifacts + - name: Regenerate dbt artifacts and run `dbt-bouncer` + if: matrix.operating-system == 'windows-latest' run: | + $env:Path += ';C:\Users\runneradmin\.local\bin' poetry run dbt deps poetry run dbt build poetry run dbt docs generate + poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml - - name: Run `dbt-bouncer` on generated artifacts - run: poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml + - name: Regenerate dbt artifacts and run `dbt-bouncer` + if: matrix.operating-system != 'windows-latest' + run: | + poetry run dbt deps + poetry run dbt build + poetry run dbt docs generate + poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml diff --git a/.github/workflows/post_release_pipeline.yml b/.github/workflows/post_release_pipeline.yml index e45e6a74..e875a946 100644 --- a/.github/workflows/post_release_pipeline.yml +++ b/.github/workflows/post_release_pipeline.yml @@ -96,17 +96,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Determine python version - id: python-version - run: | - if [[ ${{ matrix.operating-system }} == "windows-latest" ]]; then - echo "PYTHON_VERSION=$(cat .python-version)" >> $ENV:GITHUB_OUTPUT - else - export PYTHON_VERSION=$(cat .python-version) - echo "PYTHON_VERSION: $PYTHON_VERSION" - echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_OUTPUT - fi - - name: Setup Python uses: ./.github/actions/setup_python_env with: @@ -124,11 +113,19 @@ jobs: - name: Run `dbt-bouncer` run: dbt-bouncer --config-file dbt-bouncer-example.yml - - name: Regenerate dbt artifacts + - name: Regenerate dbt artifacts and run `dbt-bouncer` + if: matrix.operating-system == 'windows-latest' run: | + $env:Path += ';C:\Users\runneradmin\.local\bin' poetry run dbt deps poetry run dbt build poetry run dbt docs generate + poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml - - name: Run `dbt-bouncer` on generated artifacts - run: poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml + - name: Regenerate dbt artifacts and run `dbt-bouncer` + if: matrix.operating-system != 'windows-latest' + run: | + poetry run dbt deps + poetry run dbt build + poetry run dbt docs generate + poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml diff --git a/makefile b/makefile index 7f670395..2c0c0dec 100644 --- a/makefile +++ b/makefile @@ -1,53 +1,58 @@ -build-artifacts: - poetry run python ./scripts/generate_artifacts.py - -build-pex: - poetry run pex . \ - --interpreter-constraint ">=3.8.1,<3.13" \ - --jobs 128 \ - --max-install-jobs 0 \ - --output-file ./dist/dbt-bouncer.pex \ - --pip-version 23.2 \ - --platform macosx_11_0_x86_64-cp-38-cp38 \ - --platform macosx_11_0_x86_64-cp-39-cp39 \ - --platform macosx_11_0_x86_64-cp-310-cp310 \ - --platform macosx_11_0_x86_64-cp-311-cp311 \ - --platform macosx_11_0_x86_64-cp-312-cp312 \ - --platform manylinux2014_x86_64-cp-38-cp38 \ - --platform manylinux2014_x86_64-cp-39-cp39 \ - --platform manylinux2014_x86_64-cp-310-cp310 \ - --platform manylinux2014_x86_64-cp-311-cp311 \ - --platform manylinux2014_x86_64-cp-312-cp312 \ - --python-shebang='/usr/bin/env python' \ - --script dbt-bouncer - -test: - $(MAKE) test-unit - $(MAKE) test-integration - -test-integration: - poetry run pytest \ - -c ./tests \ - --junitxml=coverage.xml \ - --cov-report=term-missing:skip-covered \ - --cov=src/dbt_bouncer/ \ - --numprocesses 5 \ - ./tests/integration \ - $(MAKE_ARGS) - -test-unit: - poetry run pytest \ - -c ./tests \ - --junitxml=coverage.xml \ - --cov-report=term-missing:skip-covered \ - --cov=src/dbt_bouncer/ \ - --numprocesses 5 \ - ./tests/unit \ - -m 'not not_in_parallel' && \ - poetry run pytest \ - -c ./tests \ - --junitxml=coverage.xml \ - --cov-report=term-missing:skip-covered \ - --cov=src/dbt_bouncer/ \ - --cov-append \ - -m not_in_parallel +build-artifacts: + poetry run python ./scripts/generate_artifacts.py + +build-pex: + poetry run pex . \ + --interpreter-constraint ">=3.8.1,<3.13" \ + --jobs 128 \ + --max-install-jobs 0 \ + --output-file ./dist/dbt-bouncer.pex \ + --pip-version 23.2 \ + --platform macosx_11_0_x86_64-cp-38-cp38 \ + --platform macosx_11_0_x86_64-cp-39-cp39 \ + --platform macosx_11_0_x86_64-cp-310-cp310 \ + --platform macosx_11_0_x86_64-cp-311-cp311 \ + --platform macosx_11_0_x86_64-cp-312-cp312 \ + --platform manylinux2014_x86_64-cp-38-cp38 \ + --platform manylinux2014_x86_64-cp-39-cp39 \ + --platform manylinux2014_x86_64-cp-310-cp310 \ + --platform manylinux2014_x86_64-cp-311-cp311 \ + --platform manylinux2014_x86_64-cp-312-cp312 \ + --python-shebang='/usr/bin/env python' \ + --script dbt-bouncer + +test: + $(MAKE) test-unit + $(MAKE) test-integration + +test-integration: + poetry run pytest \ + -c ./tests \ + --junitxml=coverage.xml \ + --cov-report=term-missing:skip-covered \ + --cov=src/dbt_bouncer/ \ + --numprocesses 5 \ + ./tests/integration \ + $(MAKE_ARGS) + +test-unit: + poetry run pytest \ + -c ./tests \ + --junitxml=coverage.xml \ + --cov-report=term-missing:skip-covered \ + --cov=src/dbt_bouncer/ \ + --numprocesses 5 \ + ./tests/unit \ + -m 'not not_in_parallel' && \ + poetry run pytest \ + -c ./tests \ + --junitxml=coverage.xml \ + --cov-report=term-missing:skip-covered \ + --cov=src/dbt_bouncer/ \ + --cov-append \ + -m not_in_parallel + +test-windows: + poetry run pytest -c ./tests --numprocesses 5 ./tests/unit -m 'not not_in_parallel' && \ + poetry run pytest -c ./tests -m not_in_parallel && \ + poetry run pytest -c ./tests --numprocesses 5 ./tests/integration From cc275c5529b762ce7539429fc63c0544e11aef2b Mon Sep 17 00:00:00 2001 From: pgoslatara Date: Mon, 9 Sep 2024 21:19:58 +0200 Subject: [PATCH 4/4] Adding quotes --- .github/workflows/post_release_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/post_release_pipeline.yml b/.github/workflows/post_release_pipeline.yml index e875a946..52eeeca0 100644 --- a/.github/workflows/post_release_pipeline.yml +++ b/.github/workflows/post_release_pipeline.yml @@ -104,7 +104,7 @@ jobs: - name: Install `dbt-bouncer` if: matrix.operating-system == 'windows-latest' - run: pip install dbt-bouncer=='("${{ github.ref }}" -Split "/")[2]' + run: pip install "dbt-bouncer=='("${{ github.ref }}" -Split "/")[2]'" - name: Install `dbt-bouncer` if: matrix.operating-system != 'windows-latest'