Symlink dbt packages when RenderConfig.dbt_deps=False (#730)

Closes: #727

cosmos/dbt/graph.py

@@ -232,7 +232,7 @@ def load_via_dbt_ls(self) -> None:
                 f"Content of the dbt project dir {self.render_config.project_path}: `{os.listdir(self.render_config.project_path)}`"
             )
             tmpdir_path = Path(tmpdir)
-            create_symlinks(self.render_config.project_path, tmpdir_path)
+            create_symlinks(self.render_config.project_path, tmpdir_path, self.render_config.dbt_deps)
 
             with self.profile_config.ensure_profile(use_mock_values=True) as profile_values, environ(
                 self.render_config.env_vars

cosmos/dbt/project.py

@@ -10,9 +10,12 @@
 from typing import Generator
 
 
-def create_symlinks(project_path: Path, tmp_dir: Path) -> None:
+def create_symlinks(project_path: Path, tmp_dir: Path, ignore_dbt_packages: bool) -> None:
     """Helper function to create symlinks to the dbt project files."""
-    ignore_paths = (DBT_LOG_DIR_NAME, DBT_TARGET_DIR_NAME, "dbt_packages", "profiles.yml")
+    ignore_paths = [DBT_LOG_DIR_NAME, DBT_TARGET_DIR_NAME, "profiles.yml"]
+    if ignore_dbt_packages:
+        # this is linked to dbt deps so if dbt deps is true then ignore existing dbt_packages folder
+        ignore_paths.append("dbt_packages")
     for child_name in os.listdir(project_path):
         if child_name not in ignore_paths:
             os.symlink(project_path / child_name, tmp_dir / child_name)

cosmos/operators/local.py

@@ -199,7 +199,7 @@ def run_command(
                 self.project_dir,
             )
 
-            create_symlinks(Path(self.project_dir), Path(tmp_project_dir))
+            create_symlinks(Path(self.project_dir), Path(tmp_project_dir), self.install_deps)
 
             with self.profile_config.ensure_profile() as profile_values:
                 (profile_path, env_vars) = profile_values

tests/dbt/test_graph.py

@@ -16,6 +16,7 @@
     run_command,
 )
 from cosmos.profiles import PostgresUserPasswordProfileMapping
+from subprocess import Popen, PIPE
 
 DBT_PROJECTS_ROOT_DIR = Path(__file__).parent.parent.parent / "dev/dags/dbt"
 DBT_PROJECT_NAME = "jaffle_shop"
@@ -436,6 +437,50 @@ def test_load_via_dbt_ls_without_dbt_deps():
     assert err_info.value.args[0] == expected
 
 
+@pytest.mark.integration
+def test_load_via_dbt_ls_without_dbt_deps_and_preinstalled_dbt_packages():
+    local_flags = [
+        "--project-dir",
+        DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME,
+        "--profiles-dir",
+        DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME,
+        "--profile",
+        "default",
+        "--target",
+        "dev",
+    ]
+
+    deps_command = ["dbt", "deps"]
+    deps_command.extend(local_flags)
+    process = Popen(
+        deps_command,
+        stdout=PIPE,
+        stderr=PIPE,
+        cwd=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME,
+        universal_newlines=True,
+    )
+    stdout, stderr = process.communicate()
+
+    project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME)
+    render_config = RenderConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME, dbt_deps=False)
+    execution_config = ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME)
+    dbt_graph = DbtGraph(
+        project=project_config,
+        render_config=render_config,
+        execution_config=execution_config,
+        profile_config=ProfileConfig(
+            profile_name="default",
+            target_name="default",
+            profile_mapping=PostgresUserPasswordProfileMapping(
+                conn_id="airflow_db",
+                profile_args={"schema": "public"},
+            ),
+        ),
+    )
+
+    dbt_graph.load_via_dbt_ls()  # does not raise exception
+
+
 @pytest.mark.integration
 @patch("cosmos.dbt.graph.Popen")
 def test_load_via_dbt_ls_with_zero_returncode_and_non_empty_stderr(mock_popen, tmp_dbt_project_dir):

tests/dbt/test_project.py

@@ -11,7 +11,7 @@ def test_create_symlinks(tmp_path):
     tmp_dir = tmp_path / "dbt-project"
     tmp_dir.mkdir()
 
-    create_symlinks(DBT_PROJECTS_ROOT_DIR / "jaffle_shop", tmp_dir)
+    create_symlinks(DBT_PROJECTS_ROOT_DIR / "jaffle_shop", tmp_dir, False)
     for child in tmp_dir.iterdir():
         assert child.is_symlink()
         assert child.name not in ("logs", "target", "profiles.yml", "dbt_packages")