Migrate Off Circle CI / To Github Actions + dagger.io #43
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Runs integration tests. | |
| # **why?** | |
| # Ensure code runs as expected. | |
| # **when?** | |
| # This will run for all PRs, when code is pushed to a release | |
| # branch, and when manually triggered. | |
| name: Integration tests | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "*.latest" | |
| - "releases/*" | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| dbt-core-branch: | |
| description: "branch of dbt-core to use in dev-requirements.txt" | |
| required: false | |
| type: string | |
| # explicitly turn off permissions for `GITHUB_TOKEN` | |
| permissions: read-all | |
| # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| # generate test metadata about what files changed and the testing matrix to use | |
| test-metadata: | |
| # run if not a PR from a forked repository or has a label to mark as safe to test | |
| if: >- | |
| github.event_name != 'pull_request' || | |
| github.event.pull_request.head.repo.full_name == github.repository || | |
| contains(github.event.pull_request.labels.*.name, 'ok to test') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.generate-matrix.outputs.result }} | |
| run-python-tests: ${{ steps.filter.outputs.bigquery-python }} | |
| steps: | |
| - name: Check out the repository (non-PR) | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Check out the repository (PR) | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Check if relevant files changed | |
| if: github.event_name == 'pull_request' | |
| # https://github.com/marketplace/actions/paths-changes-filter | |
| # For each filter, it sets output variable named by the filter to the text: | |
| # 'true' - if any of changed files matches any of filter rules | |
| # 'false' - if none of changed files matches any of filter rules | |
| # also, returns: | |
| # `changes` - JSON array with names of all filters matching any of the changed files | |
| uses: dorny/paths-filter@v2 | |
| id: get-changes | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| filters: | | |
| spark: | |
| - 'dbt/**' | |
| - 'tests/**' | |
| - 'dev-requirements.txt' | |
| local-tests: | |
| name: test spark local against python ${{ matrix.python-version }} | |
| # run if not a PR from a forked repository or has a label to mark as safe to test | |
| # also checks that the matrix generated is not empty | |
| if: >- | |
| ( | |
| github.event_name != 'pull_request' || | |
| github.event.pull_request.head.repo.full_name == github.repository || | |
| contains(github.event.pull_request.labels.*.name, 'ok to test') | |
| ) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.8" | |
| test: | |
| - "spark-thrift" | |
| - "spark-session" | |
| env: | |
| PYTEST_ADDOPTS: "-v --color=yes --csv test_results.csv" | |
| DBT_INVOCATION_ENV: github-actions | |
| DD_CIVISIBILITY_AGENTLESS_ENABLED: true | |
| DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} | |
| DD_SITE: datadoghq.com | |
| DD_ENV: ci | |
| DD_SERVICE: ${{ github.event.repository.name }} | |
| steps: | |
| - name: Check out the repository | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| # explicity checkout the branch for the PR, | |
| # this is necessary for the `pull_request` event | |
| - name: Check out the repository (PR) | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip install tox | |
| python -m pip --version | |
| tox --version | |
| - name: Update dev_requirements.txt | |
| if: inputs.dbt-core-branch != '' | |
| run: | | |
| pip install bumpversion | |
| ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} | |
| - uses: isbang/[email protected] | |
| if: ${{ matrix.test == 'spark-thrift'}} | |
| with: | |
| compose-file: "./docker-compose.yml" | |
| - name: Run tox for Spark ${{ matrix.test }} | |
| run: tox -e integration-${{ matrix.test }} | |
| databricks-tests: | |
| name: run ${{ matrix.test }} against python ${{ matrix.python-version }} | |
| # run if not a PR from a forked repository or has a label to mark as safe to test | |
| # also checks that the matrix generated is not empty | |
| if: >- | |
| ( | |
| github.event_name != 'pull_request' || | |
| github.event.pull_request.head.repo.full_name == github.repository || | |
| contains(github.event.pull_request.labels.*.name, 'ok to test') | |
| ) | |
| runs-on: ubuntu-latest | |
| container: | |
| image: "fishtownanalytics/test-container:10" | |
| options: --user root --platform linux/amd64 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.8" | |
| test: | |
| - "spark-databricks-odbc-sql-endpoint" | |
| - "spark-databricks-odbc-cluster" | |
| - "spark-databricks-http" | |
| env: | |
| PYTEST_ADDOPTS: "-v --color=yes --csv test_results.csv" | |
| DBT_INVOCATION_ENV: github-actions | |
| DD_CIVISIBILITY_AGENTLESS_ENABLED: true | |
| DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} | |
| DD_SITE: datadoghq.com | |
| DD_ENV: ci | |
| DD_SERVICE: ${{ github.event.repository.name }} | |
| DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} | |
| DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST }} | |
| DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} | |
| DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} | |
| ODBC_DRIVER: "Simba" | |
| steps: | |
| - name: Check out the repository | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| # explicity checkout the branch for the PR, | |
| # this is necessary for the `pull_request` event | |
| - name: Check out the repository (PR) | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip install tox | |
| python -m pip --version | |
| tox --version | |
| - name: Update dev_requirements.txt | |
| if: inputs.dbt-core-branch != '' | |
| run: | | |
| pip install bumpversion | |
| ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} | |
| - name: Configure ODBC | |
| if: ${{ matrix.test != 'spark-databricks-http' }} | |
| shell: bash | |
| run: | | |
| apt-get update && apt-get install -y --no-install-recommends \ | |
| g++ \ | |
| unixodbc-dev \ | |
| unzip | |
| apt-get install -y curl | |
| rm -rf /tmp && mkdir /tmp | |
| curl -OL "https://databricks.com/wp-content/uploads/drivers-2020/SimbaSparkODBC-2.6.16.1019-Debian-64bit.zip" \ | |
| unzip SimbaSparkODBC-2.6.16.1019-Debian-64bit.zip -d /tmp/ \ | |
| && dpkg -i /tmp/SimbaSparkODBC-*/*.deb \ | |
| && echo "[Simba]\nDriver = /opt/simba/spark/lib/64/libsparkodbc_sb64.so" >> /etc/odbcinst.ini \ | |
| && rm -rf /tmp \ | |
| && dpkg -l | grep Simba # confirm that the driver is installed | |
| ldd /opt/simba/spark/lib/64/libsparkodbc_sb64.so | |
| - name: Run tox for Spark ${{ matrix.test }} | |
| run: tox -e integration-${{ matrix.test }} |