dbt-core@main with dbt-common branch #1060
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Runs all tests in dbt-core with this branch of dbt-common to ensure nothing is broken | |
| # **why?** | |
| # Ensure dbt-common changes do not break dbt-core | |
| # **when?** | |
| # This will run when trying to merge a PR into main. | |
| # It can also be manually triggered. | |
| # This workflow can be skipped by adding the "Skip Core Testing" label to the PR. This is | |
| # useful when making a change in both `dbt-core` and `dbt-common` where the changes are dependant | |
| # and cause the other repository to break. | |
| name: "dbt-core Tests" | |
| run-name: >- | |
| ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') | |
| && format('dbt-core@{0} with dbt-common@{1}', inputs.dbt-core-ref, inputs.dbt-common-ref) | |
| || 'dbt-core@main with dbt-common branch' }} | |
| on: | |
| merge_group: | |
| types: [checks_requested] | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| default: "main" | |
| workflow_call: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| permissions: read-all | |
| # will cancel previous workflows triggered by the same event | |
| # and for the same ref for PRs/merges or same SHA otherwise | |
| # and for the same inputs on workflow_dispatch or workflow_call | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(fromJson('["pull_request", "merge_group"]'), github.event_name) && github.event.pull_request.head.ref || github.sha }}-${{ contains(fromJson('["workflow_call", "workflow_dispatch"]'), github.event_name) && github.event.inputs.dbt-core-ref && github.event.inputs.dbt-common-ref || github.sha }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| # top-level adjustments can be made here | |
| env: | |
| # number of parallel processes to spawn for python integration testing | |
| PYTHON_INTEGRATION_TEST_WORKERS: 5 | |
| jobs: | |
| job-prep: | |
| # This allow us to run the workflow on pull_requests as well so we can always run unit tests | |
| # and only run integration tests on merge for time purposes | |
| name: Setup Repo Refs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| dbt-core-ref: ${{ steps.core-ref.outputs.ref }} | |
| dbt-common-ref: ${{ steps.common-ref.outputs.ref }} | |
| steps: | |
| - name: "Input Refs" | |
| id: job-inputs | |
| run: | | |
| echo "inputs.dbt-core-ref=${{ inputs.dbt-core-ref }}" | |
| echo "inputs.dbt-common-ref=${{ inputs.dbt-common-ref }}" | |
| - name: "Determine dbt-core ref" | |
| id: core-ref | |
| run: | | |
| if [[ -z "${{ inputs.dbt-core-ref }}" ]]; then | |
| REF="main" | |
| else | |
| REF=${{ inputs.dbt-core-ref }} | |
| fi | |
| echo "ref=$REF" >> $GITHUB_OUTPUT | |
| - name: "Determine dbt-common ref" | |
| id: common-ref | |
| run: | | |
| if [[ -z "${{ inputs.dbt-common-ref }}" ]]; then | |
| # these will be commits instead of branches | |
| if [[ "${{ github.event_name }}" == "merge_group" ]]; then | |
| REF=${{ github.event.merge_group.head_sha }} | |
| else | |
| REF=${{ github.event.pull_request.base.sha }} | |
| fi | |
| else | |
| REF=${{ inputs.dbt-common-ref }} | |
| fi | |
| echo "ref=$REF" >> $GITHUB_OUTPUT | |
| - name: "Final Refs" | |
| run: | | |
| echo "dbt-core-ref=${{ steps.core-ref.outputs.ref }}" | |
| echo "dbt-common-ref=${{ steps.common-ref.outputs.ref }}" | |
| dbt-core-unit-test: | |
| name: "dbt-core unit tests" | |
| needs: [job-prep] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: "Check out dbt-core@${{ needs.job-prep.outputs.dbt-core-ref }}" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/dbt-core | |
| ref: ${{ needs.job-prep.outputs.dbt-core-ref }} | |
| - name: "Set up Python 3.11" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: "Upgrade pip, install tox" | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip --version | |
| python -m pip install tox | |
| tox --version | |
| - name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}" | |
| run: | | |
| ./scripts/update_dev_packages.sh ${{ github.repository.name}} ${{ needs.job-prep.outputs.dbt-common-ref }} | |
| - name: "Run unit tests" | |
| # Doing the check here instead of the top level because this is job a required check, the | |
| # label just means we should skip the tests | |
| if: ${{ !contains(github.event.label.name, 'Skip Core Testing')}} | |
| run: tox | |
| env: | |
| TOXENV: unit | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| dbt-core-integration-metadata: | |
| name: "integration test metadata generation" | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name != 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}} | |
| outputs: | |
| split-groups: ${{ steps.generate-split-groups.outputs.split-groups }} | |
| include: ${{ steps.generate-include.outputs.include }} | |
| steps: | |
| - name: "generate split-groups" | |
| id: generate-split-groups | |
| run: | | |
| MATRIX_JSON="[" | |
| for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}") | |
| done | |
| MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}" | |
| MATRIX_JSON+="]" | |
| echo "split-groups=${MATRIX_JSON}" | |
| echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT | |
| - name: "generate include" | |
| id: generate-include | |
| run: | | |
| INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-12"' ) | |
| INCLUDE_GROUPS="[" | |
| for include in ${INCLUDE[@]}; do | |
| for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}") | |
| done | |
| done | |
| INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g') | |
| INCLUDE_GROUPS+="]" | |
| echo "include=${INCLUDE_GROUPS}" | |
| echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT | |
| dbt-core-integration-tests: | |
| name: "(${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}" | |
| if: ${{ github.event_name != 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| needs: [job-prep, dbt-core-integration-metadata] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| os: [ubuntu-20.04] | |
| split-group: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.split-groups) }} | |
| include: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.include) }} | |
| env: | |
| DBT_INVOCATION_ENV: github-actions | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| steps: | |
| - name: "Check out the repository@${{ needs.job-prep.outputs.dbt-core-ref }}" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/dbt-core | |
| ref: ${{ needs.job-prep.outputs.dbt-core-ref }} | |
| - name: "Set up Python ${{ matrix.python-version }}" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: "Set up postgres (linux)" | |
| if: runner.os == 'Linux' | |
| uses: ./.github/actions/setup-postgres-linux | |
| - name: "Set up postgres (macos)" | |
| if: runner.os == 'macOS' | |
| uses: ./.github/actions/setup-postgres-macos | |
| - name: "Set up postgres (windows)" | |
| if: runner.os == 'Windows' | |
| uses: ./.github/actions/setup-postgres-windows | |
| - name: "Upgrade pip, install tox" | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip --version | |
| python -m pip install tox | |
| tox --version | |
| - name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}" | |
| run: | | |
| ./scripts/update_dev_packages.sh ${{ github.repository.name}} ${{ needs.job-prep.outputs.dbt-common-ref }} | |
| - name: "Run Functional tests" | |
| run: tox -- --ddtrace | |
| env: | |
| TOXENV: integration | |
| DBT_INVOCATION_ENV: github-actions | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| DD_CIVISIBILITY_AGENTLESS_ENABLED: true | |
| DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} | |
| DD_SITE: datadoghq.com | |
| DD_ENV: ci | |
| DD_SERVICE: dbt-core | |
| PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }} | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| integration-report: | |
| if: ${{ always() }} | |
| name: "dbt-core Integration Test Suite Report" | |
| runs-on: ubuntu-latest | |
| needs: [dbt-core-integration-tests] | |
| steps: | |
| - name: "Integration Tests Failed" | |
| if: ${{ contains(needs.dbt-core-integration-tests.result, 'failure') || contains(needs.dbt-core-integration-tests.result, 'cancelled') }} | |
| # when this is true the next step won't execute | |
| run: | | |
| echo "::notice title='Integration test suite failed'" | |
| exit 1 | |
| - name: "Integration Tests Passed" | |
| if: ${{ github.event_name != 'pull_request' }} | |
| run: | | |
| echo "::notice title='Integration test suite passed'" | |
| - name: "Integration Tests Skipped on Pull Request" | |
| if: ${{ github.event_name == 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}} | |
| run: | | |
| echo "::notice title='Integration test suite skipped on Pull Requests - they will run on merge'" | |
| - name: "Integration Tests Skipped by Label" | |
| if: ${{ contains(github.event.label.name, 'Skip Core Testing')}} | |
| run: | | |
| echo "::notice title='dbt-core test suite skipped due to Skip Core Testing label'" |