Set default value for append_env based on execution modes (#954)

This PR sets the default value of dbt argument `append-env` that 
exposes the operating system environment variables to the 
`dbt` command based on execution modes.
For most execution modes, the default is set to `False`, 
however, for execution modes `ExecuteMode.LOCAL` and 
`ExecuteMode.VIRTUALENV`, the default is set to True. 
This behavior is consistent with the `LoadMode.DBT_LS` 
command in forwarding the environment variables to the 
subprocess by default.

---------

Co-authored-by: Pankaj Singh <[email protected]>

cosmos/operators/base.py

@@ -46,10 +46,10 @@ class AbstractDbtBaseOperator(BaseOperator, metaclass=ABCMeta):
         environment variables for the new process; these are used instead
         of inheriting the current process environment, which is the default
         behavior. (templated)
-    :param append_env: . If True (default), inherits the environment variables
-        from current passes and then environment variable passed by the user will either update the existing
+    :param append_env: If True, inherits the environment variables
+        from current process and then environment variable passed by the user will either update the existing
         inherited environment variables or the new variables gets appended to it.
-        If False, only uses the environment variables passed in env params
+        If False (default), only uses the environment variables passed in env params
         and does not inherit the current process environment.
     :param output_encoding: Output encoding of bash command
     :param skip_exit_code: If task exits with this exit code, leave the task
@@ -104,7 +104,7 @@ def __init__(
         db_name: str | None = None,
         schema: str | None = None,
         env: dict[str, Any] | None = None,
-        append_env: bool = True,
+        append_env: bool = False,
         output_encoding: str = "utf-8",
         skip_exit_code: int = 99,
         partial_parse: bool = True,

cosmos/operators/local.py

@@ -113,6 +113,11 @@ class DbtLocalBaseOperator(AbstractDbtBaseOperator):
     :param target_name: A name to use for the dbt target. If not provided, and no target is found
         in your project's dbt_project.yml, "cosmos_target" is used.
     :param should_store_compiled_sql: If true, store the compiled SQL in the compiled_sql rendered template.
+    :param append_env: If True(default), inherits the environment variables
+        from current process and then environment variable passed by the user will either update the existing
+        inherited environment variables or the new variables gets appended to it.
+        If False, only uses the environment variables passed in env params
+        and does not inherit the current process environment.
     """
 
     template_fields: Sequence[str] = AbstractDbtBaseOperator.template_fields + ("compiled_sql",)  # type: ignore[operator]
@@ -127,6 +132,7 @@ def __init__(
         install_deps: bool = False,
         callback: Callable[[str], None] | None = None,
         should_store_compiled_sql: bool = True,
+        append_env: bool = True,
         **kwargs: Any,
     ) -> None:
         self.profile_config = profile_config
@@ -144,6 +150,12 @@ def __init__(
         kwargs.pop("full_refresh", None)  # usage of this param should be implemented in child classes
         super().__init__(**kwargs)
 
+        # For local execution mode, we're consistent with the LoadMode.DBT_LS command in forwarding the environment
+        # variables to the subprocess by default. Although this behavior is designed for ExecuteMode.LOCAL and
+        # ExecuteMode.VIRTUALENV, it is not desired for the other execution modes to forward the environment variables
+        # as it can break existing DAGs.
+        self.append_env = append_env
+
     @cached_property
     def subprocess_hook(self) -> FullOutputSubprocessHook:
         """Returns hook for running the bash command."""

docs/configuration/operator-args.rst

@@ -43,7 +43,7 @@ Summary of Cosmos-specific arguments
 dbt-related
 ...........
 
-- ``append_env``: Expose the operating system environment variables to the ``dbt`` command. The default is ``False``.
+- ``append_env``: Expose the operating system environment variables to the ``dbt`` command. For most execution modes, the default is ``False``, however, for execution modes ExecuteMode.LOCAL and ExecuteMode.VIRTUALENV, the default is True. This behavior is consistent with the LoadMode.DBT_LS command in forwarding the environment variables to the subprocess by default.
 - ``dbt_cmd_flags``: List of command flags to pass to ``dbt`` command, added after dbt subcommand
 - ``dbt_cmd_global_flags``: List of ``dbt`` `global flags <https://docs.getdbt.com/reference/global-configs/about-global-configs>`_ to be passed to the ``dbt`` command, before the subcommand
 - ``dbt_executable_path``: Path to dbt executable.

tests/operators/test_azure_container_instance.py

@@ -53,7 +53,6 @@ def test_dbt_azure_container_instance_operator_get_env(p_context_to_airflow_vars
         name="my-aci",
         resource_group="my-rg",
         project_dir="my/dir",
-        append_env=False,
     )
     dbt_base_operator.env = {
         "start_date": "20220101",
@@ -91,7 +90,6 @@ def test_dbt_azure_container_instance_operator_check_environment_variables(
         resource_group="my-rg",
         project_dir="my/dir",
         environment_variables={"FOO": "BAR"},
-        append_env=False,
     )
     dbt_base_operator.env = {
         "start_date": "20220101",

tests/operators/test_base.py

@@ -69,3 +69,9 @@ def test_dbt_mixin_add_cmd_flags_run_operator(args, expected_flags):
 
     flags = run_operation.add_cmd_flags()
     assert flags == expected_flags
+
+
+def test_abstract_dbt_base_operator_append_env_is_false_by_default():
+    """Tests that the append_env attribute is set to False by default."""
+    base_operator = AbstractDbtBaseOperator(task_id="fake_task", project_dir="fake_dir")
+    assert base_operator.append_env is False

tests/operators/test_local.py

@@ -103,6 +103,22 @@ def test_dbt_base_operator_add_global_flags() -> None:
     ]
 
 
+def test_dbt_local_operator_append_env_is_true_by_default() -> None:
+    dbt_local_operator = ConcreteDbtLocalBaseOperator(
+        profile_config=profile_config,
+        task_id="my-task",
+        project_dir="my/dir",
+        vars={
+            "start_time": "{{ data_interval_start.strftime('%Y%m%d%H%M%S') }}",
+            "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}",
+        },
+        no_version_check=True,
+        select=["my_first_model", "my_second_model"],
+    )
+
+    assert dbt_local_operator.append_env == True
+
+
 def test_dbt_base_operator_add_user_supplied_flags() -> None:
     dbt_base_operator = ConcreteDbtLocalBaseOperator(
         profile_config=profile_config,

tests/operators/test_virtualenv.py

@@ -73,3 +73,20 @@ def test_run_command(
     assert dbt_deps["command"][0] == dbt_cmd["command"][0]
     assert dbt_cmd["command"][1] == "do-something"
     assert mock_execute.call_count == 2
+
+
+def test_virtualenv_operator_append_env_is_true_by_default():
+    venv_operator = ConcreteDbtVirtualenvBaseOperator(
+        dag=DAG("sample_dag", start_date=datetime(2024, 4, 16)),
+        profile_config=profile_config,
+        task_id="fake_task",
+        install_deps=True,
+        project_dir="./dev/dags/dbt/jaffle_shop",
+        py_system_site_packages=False,
+        pip_install_options=["--test-flag"],
+        py_requirements=["dbt-postgres==1.6.0b1"],
+        emit_datasets=False,
+        invocation_mode=InvocationMode.SUBPROCESS,
+    )
+
+    assert venv_operator.append_env is True