diff --git a/dev/dags/cosmos_manifest_example.py b/dev/dags/cosmos_manifest_example.py index 543fe641c..8e35208b8 100644 --- a/dev/dags/cosmos_manifest_example.py +++ b/dev/dags/cosmos_manifest_example.py @@ -87,26 +87,24 @@ def cosmos_manifest_example() -> None: # [END gcp_gs_example] # [START azure_abfs_example] - # azure_abfs_example = DbtTaskGroup( - # group_id="azure_abfs_example", - # project_config=ProjectConfig( - # manifest_path="abfs://cosmos-manifest-test/manifest.json", - # manifest_conn_id="azure_abfs_conn", - # # `manifest_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. - # project_name="jaffle_shop", - # ), - # profile_config=profile_config, - # render_config=render_config, - # execution_config=execution_config, - # operator_args={"install_deps": True}, - # ) + azure_abfs_example = DbtTaskGroup( + group_id="azure_abfs_example", + project_config=ProjectConfig( + manifest_path="abfs://cosmos-manifest-test/manifest.json", + manifest_conn_id="azure_abfs_conn", + # `manifest_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. + project_name="jaffle_shop", + ), + profile_config=profile_config, + render_config=render_config, + execution_config=execution_config, + operator_args={"install_deps": True}, + ) # [END azure_abfs_example] post_dbt = EmptyOperator(task_id="post_dbt") - (pre_dbt >> local_example >> aws_s3_example >> gcp_gs_example >> post_dbt) - # TODO: re-enable the following - # (pre_dbt >> local_example >> aws_s3_example >> gcp_gs_example >> azure_abfs_example >> post_dbt) + (pre_dbt >> local_example >> aws_s3_example >> gcp_gs_example >> azure_abfs_example >> post_dbt) cosmos_manifest_example() diff --git a/pyproject.toml b/pyproject.toml index 912dd83ea..11290b1a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ amazon = [ "apache-airflow-providers-amazon[s3fs]>=3.0.0", ] google = ["apache-airflow-providers-google>=10.17.0"] -microsoft = ["apache-airflow-providers-microsoft-azure"] +microsoft = ["apache-airflow-providers-microsoft-azure>=8.5.0"] all = [ "astronomer-cosmos[dbt-all]", "astronomer-cosmos[openlineage]", @@ -95,7 +95,7 @@ aws_eks = [ "apache-airflow-providers-amazon>=8.0.0", ] azure-container-instance = [ - "apache-airflow-providers-microsoft-azure>=8.4.0", + "apache-airflow-providers-microsoft-azure>=8.5.0", ] gcp-cloud-run-job = [ "apache-airflow-providers-google>=10.11.0", @@ -187,7 +187,7 @@ dependencies = [ "apache-airflow-providers-amazon[s3fs]>=3.0.0", "apache-airflow-providers-cncf-kubernetes>=5.1.1", "apache-airflow-providers-google>=10.17.0", - "apache-airflow-providers-microsoft-azure", + "apache-airflow-providers-microsoft-azure>=8.5.0", "msgpack", "openlineage-airflow", "pydantic>=1.10.0", diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh index 7017fc628..da8e72fe3 100755 --- a/scripts/test/pre-install-airflow.sh +++ b/scripts/test/pre-install-airflow.sh @@ -44,6 +44,7 @@ elif [ "$AIRFLOW_VERSION" = "2.7" ] ; then else uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt + # The Airflow 2.9 constraints file at # https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.11.txt # specifies apache-airflow-providers-google==10.16.0. However, our CI setup uses a Google connection without a token, @@ -52,7 +53,14 @@ else # we are using apache-airflow-providers-google>=10.17.0 and skipping constraints installation, as the specified # version does not meet our requirements. uv pip install "apache-airflow-providers-google>=10.17.0" - uv pip install apache-airflow-providers-microsoft-azure --constraint /tmp/constraint.txt + + # The Airflow 2.8 constraints file at + # https://raw.githubusercontent.com/apache/airflow/constraints-2.8.0/constraints-3.11.txt + # specifies apache-airflow-providers-microsoft-azure==8.4.0. However, our Azure connection setup in the CI, + # previously led to authentication issues with this version. This issue got resolved in + # apache-airflow-providers-microsoft-azure==8.5.0. Hence, we are using apache-airflow-providers-microsoft-azure>=8.5.0 + # and skipping installation with constraints, as the specified version does not meet our requirements. + uv pip install "apache-airflow-providers-microsoft-azure>=8.5.0" fi rm /tmp/constraint.txt