Skip to content

Merge branch 'master' into novakjiri-PST-2117-target #781

Merge branch 'master' into novakjiri-PST-2117-target

Merge branch 'master' into novakjiri-PST-2117-target #781

Workflow file for this run

name: 'GitHub Actions'
'on':
- push
concurrency: 'ci-${{ github.ref }}'
env:
APP_IMAGE: keboola/dbt-transformation
KBC_DEVELOPERPORTAL_VENDOR: keboola
KBC_DEVELOPERPORTAL_APP: keboola.dbt-transformation
KBC_DEVELOPERPORTAL_USERNAME: keboola+dbt_transformation_gha
KBC_DEVELOPERPORTAL_PASSWORD: '${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }}'
# Snowflake Credentials
SNOWFLAKE_HOST: "keboola.snowflakecomputing.com"
SNOWFLAKE_PORT: 443
SNOWFLAKE_WAREHOUSE: "KEBOOLA_PROD_SMALL"
SNOWFLAKE_DATABASE: "SAPI_9317"
SNOWFLAKE_SCHEMA: "WORKSPACE_964066099"
SNOWFLAKE_USER: "SAPI_WORKSPACE_964066099"
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
# BigQuery
BIGQUERY_PROJECT_ID: devel-249608
BIGQUERY_DATASET: dbt
BIGQUERY_THREADS: 1
BIGQUERY_LOCATION: europe-central2
BIGQUERY_KEY_CONTENT: ${{ secrets.BIGQUERY_KEY_CONTENT }}
# Redshift
REDSHIFT_DB_HOST: testing.cx4py8yq28xb.us-east-1.redshift.amazonaws.com
REDSHIFT_DB_PORT: 5439
REDSHIFT_DB_USER: dbt_redshift_ci
REDSHIFT_DB_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }}
REDSHIFT_DB_DATABASE: dbt_redshift_ci
REDSHIFT_DB_SCHEMA: public
REDSHIFT_DB_THREADS: 4
GITHUB_USERNAME: "dbt-test-project-user"
GITHUB_PASSWORD: ${{ secrets.GH_PAT }}
GITLAB_USERNAME: "dbt-test-project-user"
GITLAB_PASSWORD: ${{ secrets.GITLAB_PAT }}
BITBUCKET_USERNAME: "dbt-test-project-user"
BITBUCKET_PASSWORD: ${{ secrets.BITBUCKET_PAT }}
KBC_URL: "https://connection.keboola.com"
KBC_TOKEN: ${{ secrets.KBC_API_TOKEN }}
KBC_TOKEN_NO_RO_MAPPING: ${{ secrets.KBC_API_TOKEN_NO_RO_MAPPING }}
SNFK_KBC_URL: "https://connection.keboola.com"
SNFK_KBC_TOKEN: ${{ secrets.SNFK_KBC_TOKEN }}
GCP_KBC_TOKEN: ${{ secrets.GCP_KBC_TOKEN }}
GCP_KBC_URL: "https://connection.europe-west3.gcp.keboola.com"
# BigQuery
BQ_SCHEMA: "WORKSPACE_40431"
BQ_LOCATION: "europe-west3"
BQ_CREDENTIALS_TYPE: "service_account"
BQ_CREDENTIALS_PROJECT_ID: "kbc-euw3-55"
BQ_CREDENTIALS_PRIVATE_KEY_ID: ${{ secrets.BQ_CREDENTIALS_PRIVATE_KEY_ID }}
BQ_CREDENTIALS_PRIVATE_KEY: ${{ secrets.BQ_CREDENTIALS_PRIVATE_KEY }}
BQ_CREDENTIALS_CLIENT_EMAIL: "[email protected]"
BQ_CREDENTIALS_CLIENT_ID: "112082628224921357272"
BQ_CREDENTIALS_AUTH_URI: "https://accounts.google.com/o/oauth2/auth"
BQ_CREDENTIALS_TOKEN_URI: "https://oauth2.googleapis.com/token"
BQ_CREDENTIALS_AUTH_PROVIDER_X509_CERT_URL: "https://www.googleapis.com/oauth2/v1/certs"
BQ_CREDENTIALS_CLIENT_X509_CERT_URL: "https://www.googleapis.com/robot/v1/metadata/x509/kbc-euw3-ws-40431%40kbc-euw3-55.iam.gserviceaccount.com"
KBC_COMPONENTID: "keboola.dbt-transformation"
DOCKERHUB_USER: '${{ secrets.DOCKERHUB_USER }}'
DOCKERHUB_TOKEN: '${{ secrets.DOCKERHUB_TOKEN }}'
KBC_STORAGE_TOKEN: '${{ secrets.KBC_STORAGE_TOKEN }}'
KBC_TEST_PROJECT_URL: ''
KBC_TEST_PROJECT_CONFIGS: ''
jobs:
build:
runs-on: ubuntu-latest
outputs:
app_image_tag: '${{ steps.tag.outputs.app_image_tag }}'
is_semantic_tag: '${{ steps.tag.outputs.is_semantic_tag }}'
strategy:
matrix:
APP_ID:
- keboola.dbt-transformation
- keboola.dbt-transformation-local-bigquery
- keboola.dbt-transformation-remote-snowflake
- keboola.dbt-transformation-remote-postgresql
- keboola.dbt-transformation-remote-mssql
- keboola.dbt-transformation-remote-bigquery
- keboola.dbt-transformation-remote-redshift
dbt_version: [ 1.8.6, 1.7.1, 1.6.8, 1.5.9, 1.4.9 ]
# MSSQL is not supported yet in dbt 1.7.1 and 1.6.8
exclude:
- APP_ID: keboola.dbt-transformation-remote-mssql
dbt_version: 1.7.1
- APP_ID: keboola.dbt-transformation-remote-mssql
dbt_version: 1.6.8
steps:
-
name: 'Check out the repo'
uses: actions/checkout@v4
-
name: 'Print Docker version'
run: 'docker -v'
-
name: 'Docker login'
if: env.DOCKERHUB_TOKEN
run: 'docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN"'
-
name: 'Build image'
run: 'docker build --build-arg DBT_VERSION=${{ matrix.dbt_version }} -t $APP_IMAGE .'
-
name: 'Set image tag'
id: tag
run: |
TAG="${GITHUB_REF##*/}"
IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false)
echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'"
echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT
echo "app_image_tag=$TAG" >> $GITHUB_OUTPUT
-
name: 'Push image to ECR'
uses: keboola/action-push-to-ecr@master
with:
vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
app_id: '${{ matrix.APP_ID }}'
username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
tag: '${{ steps.tag.outputs.app_image_tag }}-${{ matrix.dbt_version }}'
push_latest: '${{ steps.tag.outputs.is_semantic_tag }}'
source_image: '${{ env.APP_IMAGE }}'
tests:
needs: build
runs-on: ubuntu-20.04
strategy:
fail-fast: false
max-parallel: 1
matrix:
dbt_version: [ 1.8.6, 1.7.1, 1.6.8, 1.5.9, 1.4.9 ]
steps:
-
name: 'Check out the repo'
uses: actions/checkout@v4
-
name: 'Pull image from ECR'
uses: keboola/action-pull-from-ecr@master
with:
vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
tag: '${{ needs.build.outputs.app_image_tag }}-${{ matrix.dbt_version }}'
target_image: '${{ env.APP_IMAGE}}'
tag_as_latest: true
-
name: 'Run tests'
run: |
docker compose run dev composer install
docker compose run wait
docker compose run \
-e SNOWFLAKE_HOST \
-e SNOWFLAKE_PORT \
-e SNOWFLAKE_WAREHOUSE \
-e SNOWFLAKE_DATABASE \
-e SNOWFLAKE_SCHEMA \
-e SNOWFLAKE_USER \
-e SNOWFLAKE_PASSWORD \
-e BIGQUERY_PROJECT_ID \
-e BIGQUERY_DATASET \
-e BIGQUERY_LOCATION \
-e BIGQUERY_THREADS \
-e BIGQUERY_KEY_CONTENT \
-e REDSHIFT_DB_HOST \
-e REDSHIFT_DB_PORT \
-e REDSHIFT_DB_USER \
-e REDSHIFT_DB_PASSWORD \
-e REDSHIFT_DB_DATABASE \
-e REDSHIFT_DB_SCHEMA \
-e REDSHIFT_DB_THREADS \
-e GITHUB_USERNAME \
-e GITHUB_PASSWORD \
-e GITLAB_USERNAME \
-e GITLAB_PASSWORD \
-e BITBUCKET_USERNAME \
-e BITBUCKET_PASSWORD \
-e KBC_URL \
-e KBC_TOKEN \
-e KBC_TOKEN_NO_RO_MAPPING \
-e SNFK_KBC_URL \
-e SNFK_KBC_TOKEN \
-e GCP_KBC_TOKEN \
-e GCP_KBC_URL \
-e BQ_SCHEMA \
-e BQ_LOCATION \
-e BQ_CREDENTIALS_TYPE \
-e BQ_CREDENTIALS_PROJECT_ID \
-e BQ_CREDENTIALS_PRIVATE_KEY_ID \
-e BQ_CREDENTIALS_PRIVATE_KEY \
-e BQ_CREDENTIALS_CLIENT_EMAIL \
-e BQ_CREDENTIALS_CLIENT_ID \
-e BQ_CREDENTIALS_AUTH_URI \
-e BQ_CREDENTIALS_TOKEN_URI \
-e BQ_CREDENTIALS_AUTH_PROVIDER_X509_CERT_URL \
-e BQ_CREDENTIALS_CLIENT_X509_CERT_URL \
-e KBC_COMPONENTID \
dev composer ci
tests-in-kbc:
needs: build
runs-on: ubuntu-latest
steps:
-
name: 'Run KBC test jobs'
if: 'env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS'
uses: keboola/action-run-configs-parallel@master
with:
token: '${{ env.KBC_STORAGE_TOKEN }}'
componentId: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
tag: '${{ needs.build.outputs.app_image_tag }}-1.5.0'
configs: '${{ env.KBC_TEST_PROJECT_CONFIGS }}'
deploy:
needs:
- build
- tests
- tests-in-kbc
runs-on: ubuntu-latest
if: 'startsWith(github.ref, ''refs/tags/'') && needs.build.outputs.is_semantic_tag == ''true'''
strategy:
matrix:
APP_ID:
- keboola.dbt-transformation
- keboola.dbt-transformation-local-bigquery
- keboola.dbt-transformation-remote-snowflake
- keboola.dbt-transformation-remote-postgresql
- keboola.dbt-transformation-remote-mssql
- keboola.dbt-transformation-remote-bigquery
- keboola.dbt-transformation-remote-redshift
steps:
- name: 'Determine tag version'
run: |
echo "DEPLOY_TAG=${{ needs.build.outputs.app_image_tag }}-1.8.6" >> $GITHUB_ENV
- name: 'Set tag in the Developer Portal'
uses: keboola/action-set-tag-developer-portal@master
with:
vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
app_id: '${{ matrix.APP_ID }}'
username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
tag: '${{ env.DEPLOY_TAG }}'