From 5f8057c976cd0b6c1da829cb67ffb12c015306dc Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Fri, 10 May 2024 21:26:26 +0530 Subject: [PATCH] Revert "Temporarily disable all tests in CI except for dbt 1.5.4 integration test" This reverts commit 13353f63d15eb572b5ea1745675cbe834826da12. --- .github/workflows/test.yml | 678 ++++++++++++++++++------------------- 1 file changed, 339 insertions(+), 339 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6b4aef553..63d3abade 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,278 +19,278 @@ jobs: steps: - run: true -# Type-Check: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha || github.ref }} -# -# - uses: actions/setup-python@v3 -# with: -# python-version: "3.9" -# architecture: "x64" -# -# - run: pip3 install hatch -# - run: hatch run tests.py3.9-2.7:type-check -# -# Run-Unit-Tests: -# runs-on: ubuntu-latest -# strategy: -# matrix: -# python-version: ["3.8", "3.9", "3.10", "3.11"] -# airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9"] -# exclude: -# - python-version: "3.11" -# airflow-version: "2.3" -# - python-version: "3.11" -# airflow-version: "2.4" -# steps: -# - uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha || github.ref }} -# -# - uses: actions/cache@v3 -# with: -# path: | -# ~/.cache/pip -# .nox -# key: unit-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} -# -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# -# - name: Install packages and dependencies -# run: | -# python -m pip install hatch -# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze -# -# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} -# run: | -# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-cov -# -# - name: Upload coverage to Github -# uses: actions/upload-artifact@v2 -# with: -# name: coverage-unit-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} -# path: .coverage -# -# Run-Integration-Tests: -# needs: Authorize -# runs-on: ubuntu-latest -# strategy: -# matrix: -# python-version: ["3.8", "3.9", "3.10", "3.11"] -# airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9"] -# exclude: -# - python-version: "3.11" -# airflow-version: "2.3" -# - python-version: "3.11" -# airflow-version: "2.4" -# services: -# postgres: -# image: postgres -# env: -# POSTGRES_PASSWORD: postgres -# options: >- -# --health-cmd pg_isready -# --health-interval 10s -# --health-timeout 5s -# --health-retries 5 -# ports: -# - 5432:5432 -# steps: -# - uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha || github.ref }} -# - uses: actions/cache@v3 -# with: -# path: | -# ~/.cache/pip -# .nox -# key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} -# -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# -# - name: Install packages and dependencies -# run: | -# python -m pip install hatch -# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze -# -# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} -# run: | -# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup -# DATABRICKS_UNIQUE_ID="${{github.run_id}}_${{matrix.python-version}}_${{ matrix.airflow-version }}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration -# env: -# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ -# AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres -# AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} -# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 -# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH -# DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} -# DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} -# DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} -# DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} -# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} -# POSTGRES_HOST: localhost -# POSTGRES_USER: postgres -# POSTGRES_PASSWORD: postgres -# POSTGRES_DB: postgres -# POSTGRES_SCHEMA: public -# POSTGRES_PORT: 5432 -# -# - name: Upload coverage to Github -# uses: actions/upload-artifact@v2 -# with: -# name: coverage-integration-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} -# path: .coverage -# -# Run-Integration-Tests-Expensive: -# needs: Authorize -# runs-on: ubuntu-latest -# strategy: -# matrix: -# python-version: ["3.11"] -# airflow-version: ["2.6"] -# -# services: -# postgres: -# image: postgres -# env: -# POSTGRES_PASSWORD: postgres -# options: >- -# --health-cmd pg_isready -# --health-interval 10s -# --health-timeout 5s -# --health-retries 5 -# ports: -# - 5432:5432 -# -# steps: -# - uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha || github.ref }} -# - uses: actions/cache@v3 -# with: -# path: | -# ~/.cache/pip -# .nox -# key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} -# -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# -# - name: Install packages and dependencies -# run: | -# python -m pip install hatch -# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze -# -# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} -# run: | -# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup -# DATABRICKS_UNIQUE_ID="${{github.run_id}}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-expensive -# env: -# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ -# AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres -# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH -# AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} -# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 -# DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} -# DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} -# DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} -# DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} -# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} -# POSTGRES_HOST: localhost -# POSTGRES_USER: postgres -# POSTGRES_PASSWORD: postgres -# POSTGRES_DB: postgres -# POSTGRES_SCHEMA: public -# POSTGRES_PORT: 5432 -# -# - name: Upload coverage to Github -# uses: actions/upload-artifact@v2 -# with: -# name: coverage-integration-expensive-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} -# path: .coverage -# -# env: -# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ -# AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres -# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH -# AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} -# DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} -# DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} -# DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} -# DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} -# -# Run-Integration-Tests-Sqlite: -# needs: Authorize -# runs-on: ubuntu-latest -# strategy: -# matrix: -# python-version: ["3.11"] -# airflow-version: ["2.7"] -# -# steps: -# - uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha || github.ref }} -# - uses: actions/cache@v3 -# with: -# path: | -# ~/.cache/pip -# .nox -# key: integration-sqlite-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} -# -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# -# - name: Install packages and dependencies -# run: | -# python -m pip install hatch -# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze -# -# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} -# run: | -# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite-setup -# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite -# env: -# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ -# AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres -# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 -# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH -# AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} -# DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} -# DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} -# DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} -# DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} -# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} -# POSTGRES_HOST: localhost -# POSTGRES_USER: postgres -# POSTGRES_PASSWORD: postgres -# POSTGRES_DB: postgres -# POSTGRES_SCHEMA: public -# POSTGRES_PORT: 5432 -# -# - name: Upload coverage to Github -# uses: actions/upload-artifact@v2 -# with: -# name: coverage-integration-sqlite-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} -# path: .coverage -# -# env: -# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ -# AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres -# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + Type-Check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + architecture: "x64" + + - run: pip3 install hatch + - run: hatch run tests.py3.9-2.7:type-check + + Run-Unit-Tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9"] + exclude: + - python-version: "3.11" + airflow-version: "2.3" + - python-version: "3.11" + airflow-version: "2.4" + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .nox + key: unit-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages and dependencies + run: | + python -m pip install hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-cov + + - name: Upload coverage to Github + uses: actions/upload-artifact@v2 + with: + name: coverage-unit-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} + path: .coverage + + Run-Integration-Tests: + needs: Authorize + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9"] + exclude: + - python-version: "3.11" + airflow-version: "2.3" + - python-version: "3.11" + airflow-version: "2.4" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .nox + key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages and dependencies + run: | + python -m pip install hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup + DATABRICKS_UNIQUE_ID="${{github.run_id}}_${{matrix.python-version}}_${{ matrix.airflow-version }}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_SCHEMA: public + POSTGRES_PORT: 5432 + + - name: Upload coverage to Github + uses: actions/upload-artifact@v2 + with: + name: coverage-integration-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} + path: .coverage + + Run-Integration-Tests-Expensive: + needs: Authorize + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + airflow-version: ["2.6"] + + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .nox + key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages and dependencies + run: | + python -m pip install hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup + DATABRICKS_UNIQUE_ID="${{github.run_id}}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-expensive + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_SCHEMA: public + POSTGRES_PORT: 5432 + + - name: Upload coverage to Github + uses: actions/upload-artifact@v2 + with: + name: coverage-integration-expensive-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} + path: .coverage + + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + + Run-Integration-Tests-Sqlite: + needs: Authorize + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + airflow-version: ["2.7"] + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .nox + key: integration-sqlite-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages and dependencies + run: | + python -m pip install hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite-setup + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_SCHEMA: public + POSTGRES_PORT: 5432 + + - name: Upload coverage to Github + uses: actions/upload-artifact@v2 + with: + name: coverage-integration-sqlite-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} + path: .coverage + + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH Run-Integration-Tests-DBT-1-5-4: needs: Authorize @@ -353,73 +353,73 @@ jobs: AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH -# Run-Performance-Tests: -# needs: Authorize -# runs-on: ubuntu-latest -# strategy: -# matrix: -# python-version: ["3.11"] -# airflow-version: ["2.7"] -# num-models: [1, 10, 50, 100] -# services: -# postgres: -# image: postgres -# env: -# POSTGRES_PASSWORD: postgres -# options: >- -# --health-cmd pg_isready -# --health-interval 10s -# --health-timeout 5s -# --health-retries 5 -# ports: -# - 5432:5432 -# steps: -# - uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha || github.ref }} -# - uses: actions/cache@v3 -# with: -# path: | -# ~/.cache/pip -# .nox -# key: perf-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} -# -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# -# - name: Install packages and dependencies -# run: | -# python -m pip install hatch -# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze -# -# - name: Run performance tests against against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} -# id: run-performance-tests -# run: | -# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance-setup -# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance -# -# # read the performance results and set them as an env var for the next step -# # format: NUM_MODELS={num_models}\nTIME={end - start}\n -# cat /tmp/performance_results.txt > $GITHUB_STEP_SUMMARY -# env: -# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ -# AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres -# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 -# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH -# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} -# POSTGRES_HOST: localhost -# POSTGRES_USER: postgres -# POSTGRES_PASSWORD: postgres -# POSTGRES_DB: postgres -# POSTGRES_SCHEMA: public -# POSTGRES_PORT: 5432 -# MODEL_COUNT: ${{ matrix.num-models }} -# env: -# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ -# AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres -# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + Run-Performance-Tests: + needs: Authorize + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + airflow-version: ["2.7"] + num-models: [1, 10, 50, 100] + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .nox + key: perf-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages and dependencies + run: | + python -m pip install hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Run performance tests against against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} + id: run-performance-tests + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance-setup + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance + + # read the performance results and set them as an env var for the next step + # format: NUM_MODELS={num_models}\nTIME={end - start}\n + cat /tmp/performance_results.txt > $GITHUB_STEP_SUMMARY + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_SCHEMA: public + POSTGRES_PORT: 5432 + MODEL_COUNT: ${{ matrix.num-models }} + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH Code-Coverage: if: github.event.action != 'labeled'