support other dbt versions #663
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'GitHub Actions' | |
| 'on': | |
| - push | |
| concurrency: 'ci-${{ github.ref }}' | |
| env: | |
| APP_IMAGE: keboola/dbt-transformation | |
| KBC_DEVELOPERPORTAL_VENDOR: keboola | |
| KBC_DEVELOPERPORTAL_APP: keboola.dbt-transformation | |
| KBC_DEVELOPERPORTAL_USERNAME: keboola+dbt_transformation_gha | |
| KBC_DEVELOPERPORTAL_PASSWORD: '${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }}' | |
| # Snowflake Credentials | |
| SNOWFLAKE_HOST: "keboola.snowflakecomputing.com" | |
| SNOWFLAKE_PORT: 443 | |
| SNOWFLAKE_WAREHOUSE: "KEBOOLA_PROD_SMALL" | |
| SNOWFLAKE_DATABASE: "SAPI_9317" | |
| SNOWFLAKE_SCHEMA: "WORKSPACE_964066099" | |
| SNOWFLAKE_USER: "SAPI_WORKSPACE_964066099" | |
| SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
| # BigQuery | |
| BIGQUERY_PROJECT_ID: devel-249608 | |
| BIGQUERY_DATASET: dbt | |
| BIGQUERY_THREADS: 1 | |
| BIGQUERY_KEY_CONTENT: ${{ secrets.BIGQUERY_KEY_CONTENT }} | |
| # Redshift | |
| REDSHIFT_DB_HOST: testing.cx4py8yq28xb.us-east-1.redshift.amazonaws.com | |
| REDSHIFT_DB_PORT: 5439 | |
| REDSHIFT_DB_USER: dbt_redshift_ci | |
| REDSHIFT_DB_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }} | |
| REDSHIFT_DB_DATABASE: dbt_redshift_ci | |
| REDSHIFT_DB_SCHEMA: public | |
| REDSHIFT_DB_THREADS: 4 | |
| GITHUB_USERNAME: "dbt-test-project-user" | |
| GITHUB_PASSWORD: ${{ secrets.GH_PAT }} | |
| GITLAB_USERNAME: "dbt-test-project-user" | |
| GITLAB_PASSWORD: ${{ secrets.GITLAB_PAT }} | |
| BITBUCKET_USERNAME: "dbt-test-project-user" | |
| BITBUCKET_PASSWORD: ${{ secrets.BITBUCKET_PAT }} | |
| KBC_URL: "https://connection.keboola.com" | |
| KBC_TOKEN: ${{ secrets.KBC_API_TOKEN }} | |
| KBC_TOKEN_NO_RO_MAPPING: ${{ secrets.KBC_API_TOKEN_NO_RO_MAPPING }} | |
| DOCKERHUB_USER: '${{ secrets.DOCKERHUB_USER }}' | |
| DOCKERHUB_TOKEN: '${{ secrets.DOCKERHUB_TOKEN }}' | |
| KBC_STORAGE_TOKEN: '${{ secrets.KBC_STORAGE_TOKEN }}' | |
| KBC_TEST_PROJECT_URL: '' | |
| KBC_TEST_PROJECT_CONFIGS: '' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| app_image_tag: '${{ steps.tag.outputs.app_image_tag }}' | |
| is_semantic_tag: '${{ steps.tag.outputs.is_semantic_tag }}' | |
| strategy: | |
| matrix: | |
| APP_ID: | |
| - keboola.dbt-transformation | |
| - keboola.dbt-transformation-remote-snowflake | |
| - keboola.dbt-transformation-remote-postgresql | |
| - keboola.dbt-transformation-remote-mssql | |
| - keboola.dbt-transformation-remote-bigquery | |
| - keboola.dbt-transformation-remote-redshift | |
| dbt_version: [ 1.7.1, 1.6.8, 1.5.9, 1.4.6, 1.3.4, 1.2.6 ] | |
| steps: | |
| - | |
| name: 'Check out the repo' | |
| uses: actions/checkout@v2 | |
| - | |
| name: 'Print Docker version' | |
| run: 'docker -v' | |
| - | |
| name: 'Docker login' | |
| if: env.DOCKERHUB_TOKEN | |
| run: 'docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN"' | |
| - | |
| name: 'Build image' | |
| run: 'docker build --build-arg DBT_VERSION=${{ matrix.dbt_version }} -t $APP_IMAGE .' | |
| - | |
| name: 'Set image tag' | |
| id: tag | |
| run: | | |
| TAG="${GITHUB_REF##*/}" | |
| IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false) | |
| echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'" | |
| echo "::set-output name=app_image_tag::$TAG" | |
| echo "::set-output name=is_semantic_tag::$IS_SEMANTIC_TAG" | |
| - | |
| name: 'Push image to ECR' | |
| uses: keboola/action-push-to-ecr@master | |
| with: | |
| vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}' | |
| app_id: '${{ matrix.APP_ID }}' | |
| username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}' | |
| password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}' | |
| tag: '${{ steps.tag.outputs.app_image_tag }}-${{ matrix.dbt_version }}' | |
| push_latest: '${{ steps.tag.outputs.is_semantic_tag }}' | |
| source_image: '${{ env.APP_IMAGE }}' | |
| tests: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| dbt_version: [ 1.7.1, 1.6.8, 1.5.9, 1.4.6, 1.3.4, 1.2.6 ] | |
| steps: | |
| - | |
| name: 'Check out the repo' | |
| uses: actions/checkout@v2 | |
| - | |
| name: 'Pull image from ECR' | |
| uses: keboola/action-pull-from-ecr@master | |
| with: | |
| vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}' | |
| app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}' | |
| username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}' | |
| password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}' | |
| tag: '${{ needs.build.outputs.app_image_tag }}-${{ matrix.dbt_version }}' | |
| target_image: '${{ env.APP_IMAGE}}' | |
| tag_as_latest: true | |
| - | |
| name: 'Run tests' | |
| run: | | |
| docker-compose run dev composer install | |
| docker-compose run wait | |
| docker-compose run \ | |
| -e SNOWFLAKE_HOST \ | |
| -e SNOWFLAKE_PORT \ | |
| -e SNOWFLAKE_WAREHOUSE \ | |
| -e SNOWFLAKE_DATABASE \ | |
| -e SNOWFLAKE_SCHEMA \ | |
| -e SNOWFLAKE_USER \ | |
| -e SNOWFLAKE_PASSWORD \ | |
| -e BIGQUERY_PROJECT_ID \ | |
| -e BIGQUERY_DATASET \ | |
| -e BIGQUERY_THREADS \ | |
| -e BIGQUERY_KEY_CONTENT \ | |
| -e REDSHIFT_DB_HOST \ | |
| -e REDSHIFT_DB_PORT \ | |
| -e REDSHIFT_DB_USER \ | |
| -e REDSHIFT_DB_PASSWORD \ | |
| -e REDSHIFT_DB_DATABASE \ | |
| -e REDSHIFT_DB_SCHEMA \ | |
| -e REDSHIFT_DB_THREADS \ | |
| -e GITHUB_USERNAME \ | |
| -e GITHUB_PASSWORD \ | |
| -e GITLAB_USERNAME \ | |
| -e GITLAB_PASSWORD \ | |
| -e BITBUCKET_USERNAME \ | |
| -e BITBUCKET_PASSWORD \ | |
| -e KBC_URL \ | |
| -e KBC_TOKEN \ | |
| -e KBC_TOKEN_NO_RO_MAPPING \ | |
| dev composer ci | |
| tests-in-kbc: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - | |
| name: 'Run KBC test jobs' | |
| if: 'env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS' | |
| uses: keboola/action-run-configs-parallel@master | |
| with: | |
| token: '${{ env.KBC_STORAGE_TOKEN }}' | |
| componentId: '${{ env.KBC_DEVELOPERPORTAL_APP }}' | |
| tag: '${{ needs.build.outputs.app_image_tag }}-1.5.0' | |
| configs: '${{ env.KBC_TEST_PROJECT_CONFIGS }}' | |
| deploy: | |
| needs: | |
| - build | |
| - tests | |
| - tests-in-kbc | |
| runs-on: ubuntu-latest | |
| if: 'startsWith(github.ref, ''refs/tags/'') && needs.build.outputs.is_semantic_tag == ''true''' | |
| strategy: | |
| matrix: | |
| APP_ID: | |
| - keboola.dbt-transformation | |
| - keboola.dbt-transformation-remote-snowflake | |
| - keboola.dbt-transformation-remote-postgresql | |
| - keboola.dbt-transformation-remote-mssql | |
| - keboola.dbt-transformation-remote-bigquery | |
| - keboola.dbt-transformation-remote-redshift | |
| steps: | |
| - | |
| name: 'Set tag in the Developer Portal' | |
| uses: keboola/action-set-tag-developer-portal@master | |
| with: | |
| vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}' | |
| app_id: '${{ matrix.APP_ID }}' | |
| username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}' | |
| password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}' | |
| tag: '${{ needs.build.outputs.app_image_tag }}-1.5.0' |