Merge branch 'main' into vertica-profile

.github/workflows/test.yml

@@ -220,6 +220,68 @@ jobs:
       DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }}
       DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
 
+  Run-Integration-Tests-Sqlite:
+    needs: Authorize
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+        airflow-version: ["2.7"]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pip
+            .nox
+          key: integration-sqlite-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install packages and dependencies
+        run: |
+          python -m pip install hatch
+          hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
+
+      - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }}
+        run: |
+          hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite-setup
+          hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite
+        env:
+          AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
+          AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:[email protected]:5432/postgres
+          PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
+          AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }}
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
+          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+          DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }}
+          DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
+          COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
+          POSTGRES_HOST: localhost
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+          POSTGRES_SCHEMA: public
+          POSTGRES_PORT: 5432
+
+      - name: Upload coverage to Github
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage-integration-sqlite-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}
+          path: .coverage
+
+    env:
+      AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
+      AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:[email protected]:5432/postgres
+      PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
+
+
   Code-Coverage:
     if: github.event.action != 'labeled'
     needs:

.gitignore

@@ -2,6 +2,9 @@
 # these files get autogenerated
 docs/profiles/*
 
+# dbt_packages is a directory that gets created when you run dbt deps
+dbt_packages/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
         types: [file]
         pass_filenames: false
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: check-added-large-files
       - id: check-merge-conflict
@@ -25,12 +25,15 @@ repos:
         args: ['--autofix']
       - id: trailing-whitespace
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
       - id: codespell
         name: Run codespell to check for common misspellings in files
         language: python
         types: [text]
+        args:
+          - --exclude-file=tests/sample/manifest_model_version.json
+          - --skip=**/manifest.json
   - repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.10.0
     hooks:
@@ -43,14 +46,14 @@ repos:
       - id: remove-tabs
         exclude: ^docs/make.bat$|^docs/Makefile$|^dev/dags/dbt/jaffle_shop/seeds/raw_orders.csv$
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.13.0
+    rev: v3.15.0
     hooks:
       - id: pyupgrade
         args:
           - --py37-plus
           - --keep-runtime-typing
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.291
+    rev: v0.0.292
     hooks:
       - id: ruff
         args:
@@ -67,7 +70,7 @@ repos:
         alias: black
         additional_dependencies: [black>=22.10.0]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.5.1'
+    rev: 'v1.6.0'
     hooks:
       - id: mypy
         name: mypy-python

CHANGELOG.rst

@@ -1,6 +1,62 @@
 Changelog
 =========
 
+1.2.0 (2023-10-13)
+------------------
+
+Features
+
+* Add support to model versioning available since dbt 1.6 by @binhnq94 in #516
+* Add AWS Athena profile mapping by @benjamin-awd in #578
+* Support customizing how dbt nodes are converted to Airflow by @tatiana in #503
+* Make the arg ``dbt_project_path`` in the ``ProjectConfig`` optional by @MrBones757 in #581
+
+Bug fixes
+
+* Fix Cosmos custom selector to support filtering a single model by @jlaneve and @harels in #576
+* Fix using ``GoogleCloudServiceAccountDictProfileMapping`` together with ``LoadMethod.DBT_LS`` by @joppevos in #587
+* Fix using the ``full_refresh`` argument in projects that contain tests by @EgorSemenov and @tatiana in #590
+* Stop creating symbolic links for ``dbt_packages`` (solves ``LocalExecutor`` concurrency issue) by @tatiana in #600
+
+Others
+
+* Docs: add reference to original Jaffle Shop project by @erdos2n in #583
+* Docs: retries & note about DagBag error by @TJaniF in #592
+* pre-commit updates in #575 and #585
+
+
+1.1.3 (2023-09-28)
+------------------
+
+Bug fixes
+
+* Only create task group and test task only if the model has a test by @raffifu in #543
+* Fix parsing test nodes when using the custom load method (LoadMethod.CUSTOM) by @raffifu in #563
+* Fix ``DbtTestOperator`` when test does not have ``test_metadata`` by @javihernovoa and @tatiana in #565
+* Support dbt 1.6 and apache-airflow-providers-cncf-kubernetes 7.3.0  by @tatiana in #564
+
+
+
+1.1.2 (2023-09-27)
+------------------
+
+Bug fixes
+
+* Fix using ``ExecutionMode.KUBERNETES`` by @pgoslatara and @tatiana in #554
+* Add support to ``apache-airflow-providers-cncf-kubernetes < 7.4.0`` by @tatiana in #553
+* Fix ``on_warning_callback`` behaviour on ``DbtTestLocalOperator`` by @edgga, @marco9663 and @tatiana in #558
+* Use ``returncode`` instead of ``stderr`` to determine dbt graph loading errors by @cliff-lau-cloverhealth in #547
+* Improve error message in ``config.py`` by @meyobagero in #532
+* Fix ``DbtTestOperator`` when test does not have ``test_metadata`` by @tatiana in #558
+* Fix ``target-path`` not specified issue in ``dbt-project.yml`` by @tatiana in #533
+
+Others
+
+* Docs: add reference links to dbt and Airflow columns by @TJaniF in #542
+* pre-commit updates #552 and #546
+
+
+
 1.1.1 (2023-09-14)
 ------------------
 

README.rst

@@ -37,7 +37,7 @@ Check out the Quickstart guide on our `docs <https://astronomer.github.io/astron
 Example Usage
 ___________________
 
-You can render an Airflow Task Group using the ``DbtTaskGroup`` class. Here's an example with the jaffle_shop project:
+You can render an Airflow Task Group using the ``DbtTaskGroup`` class. Here's an example with the `jaffle_shop project <https://github.com/dbt-labs/jaffle_shop>`_:
 
 
 .. code-block:: python
@@ -61,7 +61,7 @@ You can render an Airflow Task Group using the ``DbtTaskGroup`` class. Here's an
     with DAG(
         dag_id="extract_dag",
         start_date=datetime(2022, 11, 27),
-        schedule="@daily",
+        schedule_interval="@daily",
     ):
         e1 = EmptyOperator(task_id="pre_dbt")
 

cosmos/__init__.py

@@ -5,7 +5,7 @@
 
 Contains dags, task groups, and operators.
 """
-__version__ = "1.1.1"
+__version__ = "1.2.0"
 
 from cosmos.airflow.dag import DbtDag
 from cosmos.airflow.task_group import DbtTaskGroup
@@ -59,8 +59,8 @@
         DbtSnapshotKubernetesOperator,
         DbtTestKubernetesOperator,
     )
-except ImportError as error:
-    logger.exception(error)
+except ImportError:
+    logger.debug("To import Kubernetes modules, install astronomer-cosmos[kubernetes].", stack_info=True)
     DbtLSKubernetesOperator = MissingPackage(
         "cosmos.operators.kubernetes.DbtLSKubernetesOperator",
         "kubernetes",