Address review comments on PR 1347 regarding profile_config for Execu…

…tionMode.Docker (#1413) This PR addresses the open review comments on PR #1347 which adds profile_config to Docker execution mode. related: #1346 related: #1347 --------- Co-authored-by: Andrew Lui <[email protected]>
astronomer · Dec 20, 2024 · 2bb6043 · 2bb6043
1 parent 3b92421
commit 2bb6043
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 0 deletions.
diff --git a/cosmos/operators/docker.py b/cosmos/operators/docker.py
@@ -5,6 +5,7 @@
 from airflow.utils.context import Context
 
 from cosmos.config import ProfileConfig
+from cosmos.exceptions import CosmosValueError
 from cosmos.operators.base import (
     AbstractDbtBaseOperator,
     DbtBuildMixin,
@@ -47,6 +48,13 @@ def __init__(
         **kwargs: Any,
     ) -> None:
         self.profile_config = profile_config
+        if self.profile_config and not self.profile_config.profiles_yml_filepath:
+            raise CosmosValueError(
+                "For ExecutionMode.DOCKER, specifying ProfileConfig only works with profiles_yml_filepath method and "
+                "it must be specified. ProfileConfig with ProfileMapping method is not supported as the underlying "
+                "Airflow connections are not available in the Docker container for the mapping to work."
+            )
+
         super().__init__(image=image, **kwargs)
 
     def build_and_run_cmd(self, context: Context, cmd_flags: list[str] | None = None) -> Any:

diff --git a/docs/getting_started/docker.rst b/docs/getting_started/docker.rst
@@ -97,3 +97,15 @@ Enable and trigger a run of the `jaffle_shop_docker <https://github.com/astronom
 
 .. figure:: https://github.com/astronomer/astronomer-cosmos/raw/main/docs/_static/jaffle_shop_docker_dag_run.png
     :width: 800
+
+
+Specifying ProfileConfig
++++++++++++++++++++++++++
+
+Starting with Cosmos 1.8.0, you can use the ``profile_config`` argument in your Dbt DAG Docker operators to reference
+profiles for your dbt project defined in a profiles.yml file. To do so, provide the file’s path via the
+``profiles_yml_path`` parameter in ``profile_config``.
+
+Note that in ``ExecutionMode.DOCKER``, the ``profile_config`` is only compatible with the ``profiles_yml_path``
+approach. The ``profile_mapping`` method will not work because the required Airflow connections cannot be accessed
+within the Docker container to map them to the dbt profile.
diff --git a/tests/operators/test_docker.py b/tests/operators/test_docker.py
@@ -5,6 +5,8 @@
 from airflow.utils.context import Context
 from pendulum import datetime
 
+from cosmos import ProfileConfig
+from cosmos.exceptions import CosmosValueError
 from cosmos.operators.docker import (
     DbtBuildDockerOperator,
     DbtCloneDockerOperator,
@@ -13,6 +15,7 @@
     DbtSeedDockerOperator,
     DbtTestDockerOperator,
 )
+from cosmos.profiles import PostgresUserPasswordProfileMapping
 
 
 @pytest.fixture()
@@ -133,3 +136,27 @@ def test_dbt_docker_build_command():
             "start_time: '{{ data_interval_start.strftime(''%Y%m%d%H%M%S'') }}'\n",
             "--no-version-check",
         ]
+
+
+def test_profile_config_without_profiles_yml_raises_error(base_operator):
+    with pytest.raises(CosmosValueError) as err:
+        base_operator(
+            conn_id="my_airflow_connection",
+            task_id="my-task",
+            image="my_image",
+            project_dir="my/dir",
+            append_env=False,
+            profile_config=ProfileConfig(
+                profile_name="profile_name",
+                target_name="target_name",
+                profile_mapping=PostgresUserPasswordProfileMapping(
+                    conn_id="example_conn",
+                    profile_args={"schema": "public"},
+                ),
+            ),
+        )
+
+    error_message = str(err.value)
+    expected_err_msg = "For ExecutionMode.DOCKER, specifying ProfileConfig only works with profiles_yml_filepath method"
+
+    assert expected_err_msg in error_message