tests (run-action-with-multiple-execute-steps): fix ups try #792
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'GitHub Actions' | |
| 'on': | |
| - push | |
| concurrency: 'ci-${{ github.ref }}' | |
| env: | |
| APP_IMAGE: keboola/dbt-transformation | |
| KBC_DEVELOPERPORTAL_VENDOR: keboola | |
| KBC_DEVELOPERPORTAL_APP: keboola.dbt-transformation | |
| KBC_DEVELOPERPORTAL_USERNAME: keboola+dbt_transformation_gha | |
| KBC_DEVELOPERPORTAL_PASSWORD: '${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }}' | |
| # Snowflake Credentials | |
| SNOWFLAKE_HOST: "keboola.snowflakecomputing.com" | |
| SNOWFLAKE_PORT: 443 | |
| SNOWFLAKE_WAREHOUSE: "KEBOOLA_PROD_SMALL" | |
| SNOWFLAKE_DATABASE: "SAPI_9317" | |
| SNOWFLAKE_SCHEMA: "WORKSPACE_964066099" | |
| SNOWFLAKE_USER: "SAPI_WORKSPACE_964066099" | |
| SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
| # BigQuery | |
| BIGQUERY_PROJECT_ID: devel-249608 | |
| BIGQUERY_DATASET: dbt | |
| BIGQUERY_THREADS: 1 | |
| BIGQUERY_LOCATION: europe-central2 | |
| BIGQUERY_KEY_CONTENT: ${{ secrets.BIGQUERY_KEY_CONTENT }} | |
| # Redshift | |
| REDSHIFT_DB_HOST: testing.cx4py8yq28xb.us-east-1.redshift.amazonaws.com | |
| REDSHIFT_DB_PORT: 5439 | |
| REDSHIFT_DB_USER: dbt_redshift_ci | |
| REDSHIFT_DB_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }} | |
| REDSHIFT_DB_DATABASE: dbt_redshift_ci | |
| REDSHIFT_DB_SCHEMA: public | |
| REDSHIFT_DB_THREADS: 4 | |
| GITHUB_USERNAME: "dbt-test-project-user" | |
| GITHUB_PASSWORD: ${{ secrets.GH_PAT }} | |
| GITLAB_USERNAME: "dbt-test-project-user" | |
| GITLAB_PASSWORD: ${{ secrets.GITLAB_PAT }} | |
| BITBUCKET_USERNAME: "dbt-test-project-user" | |
| BITBUCKET_PASSWORD: ${{ secrets.BITBUCKET_PAT }} | |
| KBC_URL: "https://connection.keboola.com" | |
| KBC_TOKEN: ${{ secrets.KBC_API_TOKEN }} | |
| KBC_TOKEN_NO_RO_MAPPING: ${{ secrets.KBC_API_TOKEN_NO_RO_MAPPING }} | |
| SNFK_KBC_URL: "https://connection.keboola.com" | |
| SNFK_KBC_TOKEN: ${{ secrets.SNFK_KBC_TOKEN }} | |
| GCP_KBC_TOKEN: ${{ secrets.GCP_KBC_TOKEN }} | |
| GCP_KBC_URL: "https://connection.europe-west3.gcp.keboola.com" | |
| # BigQuery | |
| BQ_SCHEMA: "WORKSPACE_40431" | |
| BQ_LOCATION: "europe-west3" | |
| BQ_CREDENTIALS_TYPE: "service_account" | |
| BQ_CREDENTIALS_PROJECT_ID: "kbc-euw3-55" | |
| BQ_CREDENTIALS_PRIVATE_KEY_ID: ${{ secrets.BQ_CREDENTIALS_PRIVATE_KEY_ID }} | |
| BQ_CREDENTIALS_PRIVATE_KEY: ${{ secrets.BQ_CREDENTIALS_PRIVATE_KEY }} | |
| BQ_CREDENTIALS_CLIENT_EMAIL: "[email protected]" | |
| BQ_CREDENTIALS_CLIENT_ID: "112082628224921357272" | |
| BQ_CREDENTIALS_AUTH_URI: "https://accounts.google.com/o/oauth2/auth" | |
| BQ_CREDENTIALS_TOKEN_URI: "https://oauth2.googleapis.com/token" | |
| BQ_CREDENTIALS_AUTH_PROVIDER_X509_CERT_URL: "https://www.googleapis.com/oauth2/v1/certs" | |
| BQ_CREDENTIALS_CLIENT_X509_CERT_URL: "https://www.googleapis.com/robot/v1/metadata/x509/kbc-euw3-ws-40431%40kbc-euw3-55.iam.gserviceaccount.com" | |
| KBC_COMPONENTID: "keboola.dbt-transformation" | |
| DOCKERHUB_USER: '${{ secrets.DOCKERHUB_USER }}' | |
| DOCKERHUB_TOKEN: '${{ secrets.DOCKERHUB_TOKEN }}' | |
| KBC_TEST_PROJECT_URL: 'https://connection.keboola.com/admin/projects/9317' | |
| KBC_TEST_PROJECT_CONFIGS: '1192751064' | |
| GCP_KBC_TEST_PROJECT_URL: 'https://connection.europe-west3.gcp.keboola.com/admin/projects/55' | |
| GCP_KBC_TEST_PROJECT_CONFIGS: '49197' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| app_image_tag: '${{ steps.tag.outputs.app_image_tag }}' | |
| is_semantic_tag: '${{ steps.tag.outputs.is_semantic_tag }}' | |
| strategy: | |
| matrix: | |
| APP_ID: | |
| - keboola.dbt-transformation | |
| - keboola.dbt-transformation-local-bigquery | |
| - keboola.dbt-transformation-remote-snowflake | |
| - keboola.dbt-transformation-remote-postgresql | |
| - keboola.dbt-transformation-remote-mssql | |
| - keboola.dbt-transformation-remote-bigquery | |
| - keboola.dbt-transformation-remote-redshift | |
| dbt_version: [ 1.8.6, 1.7.1, 1.6.8, 1.5.9, 1.4.9 ] | |
| # MSSQL is not supported yet in dbt 1.7.1 and 1.6.8 | |
| exclude: | |
| - APP_ID: keboola.dbt-transformation-remote-mssql | |
| dbt_version: 1.7.1 | |
| - APP_ID: keboola.dbt-transformation-remote-mssql | |
| dbt_version: 1.6.8 | |
| steps: | |
| - | |
| name: 'Check out the repo' | |
| uses: actions/checkout@v4 | |
| - | |
| name: 'Print Docker version' | |
| run: 'docker -v' | |
| - | |
| name: 'Docker login' | |
| if: env.DOCKERHUB_TOKEN | |
| run: 'docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN"' | |
| - | |
| name: 'Build image' | |
| run: 'docker build --build-arg DBT_VERSION=${{ matrix.dbt_version }} -t $APP_IMAGE .' | |
| - | |
| name: 'Set image tag' | |
| id: tag | |
| run: | | |
| TAG="${GITHUB_REF##*/}" | |
| IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false) | |
| echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'" | |
| echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT | |
| echo "app_image_tag=$TAG" >> $GITHUB_OUTPUT | |
| - | |
| name: 'Push image to ECR' | |
| uses: keboola/action-push-to-ecr@master | |
| with: | |
| vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}' | |
| app_id: '${{ matrix.APP_ID }}' | |
| username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}' | |
| password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}' | |
| tag: '${{ steps.tag.outputs.app_image_tag }}-${{ matrix.dbt_version }}' | |
| push_latest: '${{ steps.tag.outputs.is_semantic_tag }}' | |
| source_image: '${{ env.APP_IMAGE }}' | |
| tests: | |
| needs: build | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| dbt_version: [ 1.8.6, 1.7.1, 1.6.8, 1.5.9, 1.4.9 ] | |
| steps: | |
| - | |
| name: 'Check out the repo' | |
| uses: actions/checkout@v4 | |
| - | |
| name: 'Pull image from ECR' | |
| uses: keboola/action-pull-from-ecr@master | |
| with: | |
| vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}' | |
| app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}' | |
| username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}' | |
| password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}' | |
| tag: '${{ needs.build.outputs.app_image_tag }}-${{ matrix.dbt_version }}' | |
| target_image: '${{ env.APP_IMAGE}}' | |
| tag_as_latest: true | |
| - | |
| name: 'Run tests' | |
| run: | | |
| docker compose run dev composer install | |
| docker compose run wait | |
| docker compose run \ | |
| -e SNOWFLAKE_HOST \ | |
| -e SNOWFLAKE_PORT \ | |
| -e SNOWFLAKE_WAREHOUSE \ | |
| -e SNOWFLAKE_DATABASE \ | |
| -e SNOWFLAKE_SCHEMA \ | |
| -e SNOWFLAKE_USER \ | |
| -e SNOWFLAKE_PASSWORD \ | |
| -e BIGQUERY_PROJECT_ID \ | |
| -e BIGQUERY_DATASET \ | |
| -e BIGQUERY_LOCATION \ | |
| -e BIGQUERY_THREADS \ | |
| -e BIGQUERY_KEY_CONTENT \ | |
| -e REDSHIFT_DB_HOST \ | |
| -e REDSHIFT_DB_PORT \ | |
| -e REDSHIFT_DB_USER \ | |
| -e REDSHIFT_DB_PASSWORD \ | |
| -e REDSHIFT_DB_DATABASE \ | |
| -e REDSHIFT_DB_SCHEMA \ | |
| -e REDSHIFT_DB_THREADS \ | |
| -e GITHUB_USERNAME \ | |
| -e GITHUB_PASSWORD \ | |
| -e GITLAB_USERNAME \ | |
| -e GITLAB_PASSWORD \ | |
| -e BITBUCKET_USERNAME \ | |
| -e BITBUCKET_PASSWORD \ | |
| -e KBC_URL \ | |
| -e KBC_TOKEN \ | |
| -e KBC_TOKEN_NO_RO_MAPPING \ | |
| -e SNFK_KBC_URL \ | |
| -e SNFK_KBC_TOKEN \ | |
| -e GCP_KBC_TOKEN \ | |
| -e GCP_KBC_URL \ | |
| -e BQ_SCHEMA \ | |
| -e BQ_LOCATION \ | |
| -e BQ_CREDENTIALS_TYPE \ | |
| -e BQ_CREDENTIALS_PROJECT_ID \ | |
| -e BQ_CREDENTIALS_PRIVATE_KEY_ID \ | |
| -e BQ_CREDENTIALS_PRIVATE_KEY \ | |
| -e BQ_CREDENTIALS_CLIENT_EMAIL \ | |
| -e BQ_CREDENTIALS_CLIENT_ID \ | |
| -e BQ_CREDENTIALS_AUTH_URI \ | |
| -e BQ_CREDENTIALS_TOKEN_URI \ | |
| -e BQ_CREDENTIALS_AUTH_PROVIDER_X509_CERT_URL \ | |
| -e BQ_CREDENTIALS_CLIENT_X509_CERT_URL \ | |
| -e KBC_COMPONENTID \ | |
| dev composer ci | |
| tests-in-kbc: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - | |
| name: 'Run KBC test jobs' | |
| if: 'env.KBC_TOKEN && env.KBC_TEST_PROJECT_CONFIGS' | |
| uses: keboola/action-run-configs-parallel@master | |
| with: | |
| token: '${{ env.KBC_TOKEN }}' | |
| componentId: '${{ env.KBC_DEVELOPERPORTAL_APP }}' | |
| tag: '${{ needs.build.outputs.app_image_tag }}-1.8.6' | |
| configs: '${{ env.KBC_TEST_PROJECT_CONFIGS }}' | |
| - | |
| name: 'Run KBC test jobs (GCP BQ)' | |
| if: 'env.GCP_KBC_TOKEN && env.GCP_KBC_TEST_PROJECT_CONFIGS' | |
| uses: keboola/action-run-configs-parallel@master | |
| with: | |
| token: '${{ env.GCP_KBC_TOKEN }}' | |
| componentId: 'keboola.dbt-transformation-local-bigquery' | |
| tag: '${{ needs.build.outputs.app_image_tag }}-1.8.6' | |
| configs: '${{ env.GCP_KBC_TEST_PROJECT_CONFIGS }}' | |
| deploy: | |
| needs: | |
| - build | |
| - tests | |
| - tests-in-kbc | |
| runs-on: ubuntu-latest | |
| if: 'startsWith(github.ref, ''refs/tags/'') && needs.build.outputs.is_semantic_tag == ''true''' | |
| strategy: | |
| matrix: | |
| APP_ID: | |
| - keboola.dbt-transformation | |
| - keboola.dbt-transformation-local-bigquery | |
| - keboola.dbt-transformation-remote-snowflake | |
| - keboola.dbt-transformation-remote-postgresql | |
| - keboola.dbt-transformation-remote-mssql | |
| - keboola.dbt-transformation-remote-bigquery | |
| - keboola.dbt-transformation-remote-redshift | |
| steps: | |
| - name: 'Determine tag version' | |
| run: | | |
| echo "DEPLOY_TAG=${{ needs.build.outputs.app_image_tag }}-1.8.6" >> $GITHUB_ENV | |
| - name: 'Set tag in the Developer Portal' | |
| uses: keboola/action-set-tag-developer-portal@master | |
| with: | |
| vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}' | |
| app_id: '${{ matrix.APP_ID }}' | |
| username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}' | |
| password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}' | |
| tag: '${{ env.DEPLOY_TAG }}' |