integration_tests.yml

name: Integration Testing

on:
  push:
    branches: [ "main" ]
  pull_request:
    branches: [ "main" ]

jobs:
  build:

    runs-on: ubuntu-latest
    environment:
      name: ci_testing
    strategy:
      fail-fast: true
      max-parallel: 1
      matrix:
        python-version: ["3.11", "3.12"]
        dbt-version: ["1.6", "1.7"]
        data-platform: ["redshift", "snowflake"]
        # python-version: ["3.9", "3.10", "3.11", "3.12"]
        # data-platform: ["redshift", "snowflake","bigquery"]

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
        # cache: 'pip'
    - name: Install dbt-${{ matrix.data-platform }}~=${{ matrix.dbt-version }}
      run: |
        python -m pip install --upgrade pip
        python -m pip install "dbt-${{ matrix.data-platform }}~=${{ matrix.dbt-version }}.0"
    - name: run integration_tests project on ${{ matrix.data-platform }}
      run: |
        cd integration_tests
        export DBT_PROFILES_DIR=.
        dbt deps --target ${{ matrix.data-platform }}
        dbt seed --full-refresh --target ${{ matrix.data-platform }}
        dbt run-operation prep_external --target ${{ matrix.data-platform }}
        dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target ${{ matrix.data-platform }}
        dbt run-operation dbt_external_tables.stage_external_sources --target ${{ matrix.data-platform }}
        dbt test --target ${{ matrix.data-platform }}
      env: 
        REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
        REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
        REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
        REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
        REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
        REDSHIFT_SPECTRUM_IAM_ROLE: ${{ secrets.REDSHIFT_SPECTRUM_IAM_ROLE }}
        SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
        SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
        SNOWFLAKE_TEST_PASS: ${{ secrets.SNOWFLAKE_TEST_PASS }}
        SNOWFLAKE_TEST_WHNAME: ${{ secrets.SNOWFLAKE_TEST_WHNAME }}
        SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
        SNOWFLAKE_TEST_DBNAME: ${{ secrets.SNOWFLAKE_TEST_DBNAME }}