From 85f4f3b33c4b37b03e1a6ab352f20b014deee9a6 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 28 Sep 2023 15:44:51 +0100 Subject: [PATCH] Support dbt 1.6 and apache-airflow-providers-cncf-kubernetes 7.3.0 (#564) Cosmos tests broke when `dbt` 1.6 and `apache-airflow-providers-cncf-kubernetes` 7.3.0 were released. There was a quick fix, #426, so the CI wouldn't be red, but that meant we didn't officially support the latest version of dbt if users wanted to run both Cosmos and dbt in the same Python virtualenv. This feature has been requested by a few community members via Airflow Slack #airflow-dbt channel: * https://apache-airflow.slack.com/archives/C059CC42E9W/p1691652944543079?thread_ts=1691650827.900949&cid=C059CC42E9W * https://apache-airflow.slack.com/archives/C059CC42E9W/p1692396096001339 Closes: #444 This change was validated by: * Running tests locally (Python 3.8 and Airflow 2.7) * Creating a virtualenv with the latest versions * Running the dags `basic_cosmos_dag` and `jaffle_shop_kubernetes` --- pyproject.toml | 32 +++++++++++++++--------------- tests/operators/test_kubernetes.py | 4 +++- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dd88d2d15..11e819774 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,34 +46,34 @@ dependencies = [ [project.optional-dependencies] dbt-all = [ - "dbt-bigquery<=1.5.4", - "dbt-databricks<=1.5.4", - "dbt-exasol<=1.5.4", - "dbt-postgres<=1.5.4", - "dbt-redshift<=1.5.4", - "dbt-snowflake<=1.5.4", - "dbt-spark<=1.5.4", + "dbt-bigquery", + "dbt-databricks", + "dbt-exasol", + "dbt-postgres", + "dbt-redshift", + "dbt-snowflake", + "dbt-spark", ] dbt-bigquery = [ - "dbt-bigquery<=1.5.4", + "dbt-bigquery", ] dbt-databricks = [ - "dbt-databricks<=1.5.4", + "dbt-databricks", ] dbt-exasol = [ - "dbt-exasol<=1.5.4", + "dbt-exasol", ] dbt-postgres = [ - "dbt-postgres<=1.5.4", + "dbt-postgres", ] dbt-redshift = [ - "dbt-redshift<=1.5.4", + "dbt-redshift", ] dbt-snowflake = [ - "dbt-snowflake<=1.5.4", + "dbt-snowflake", ] dbt-spark = [ - "dbt-spark<=1.5.4", + "dbt-spark", ] openlineage = [ "openlineage-integration-common", @@ -107,7 +107,7 @@ docker = [ "apache-airflow-providers-docker>=3.5.0", ] kubernetes = [ - "apache-airflow-providers-cncf-kubernetes>=5.1.1,<7.3.0", + "apache-airflow-providers-cncf-kubernetes>=5.1.1", ] @@ -132,7 +132,7 @@ include = [ dependencies = [ "astronomer-cosmos[tests]", "apache-airflow-providers-docker>=3.5.0", - "apache-airflow-providers-cncf-kubernetes>=5.1.1,<7.3.0", + "apache-airflow-providers-cncf-kubernetes>=5.1.1", "types-PyYAML", "types-attrs", "types-requests", diff --git a/tests/operators/test_kubernetes.py b/tests/operators/test_kubernetes.py index 839a6db54..7ef606cfe 100644 --- a/tests/operators/test_kubernetes.py +++ b/tests/operators/test_kubernetes.py @@ -171,7 +171,7 @@ def test_created_pod(test_hook): "stdin": None, "stdin_once": None, "termination_message_path": None, - "termination_message_policy": None, + # "termination_message_policy": None, "tty": None, "volume_devices": None, "volume_mounts": [], @@ -213,6 +213,8 @@ def test_created_pod(test_hook): }, "status": None, } + computed_result = pod_obj.to_dict() + computed_result["spec"]["containers"][0].pop("termination_message_policy") computed_result["metadata"].pop("namespace") assert computed_result == expected_result