added dbt_project_path_parent property to ProjectConfig

Removed nested functions from graph.load_manual
updated dags with alternate inputs to get better e2e test coverage

cosmos/config.py

@@ -73,6 +73,10 @@ def parsed_dbt_project_path(self) -> Path | None:
     def parsed_manifest_path(self) -> Path | None:
         return Path(self.manifest_path) if self.manifest_path else None
 
+    @cached_property
+    def dbt_project_path_parent(self) -> Path | None:
+        return self.parsed_dbt_project_path.parent if self.parsed_dbt_project_path else None
+
     def __post_init__(self) -> None:
         "Converts paths to `Path` objects."
         if self.parsed_dbt_project_path:

cosmos/converter.py

@@ -106,7 +106,6 @@ def __init__(
         project_config.validate_project()
 
         emit_datasets = render_config.emit_datasets
-        dbt_root_path = project_config.parsed_dbt_project_path
         dbt_project_name = project_config.project_name
         dbt_models_dir = project_config.models_relative_path
         dbt_seeds_dir = project_config.seeds_relative_path
@@ -131,7 +130,7 @@ def __init__(
 
         dbt_project = DbtProject(
             name=dbt_project_name,
-            root_dir=dbt_root_path.parent if dbt_root_path else None,
+            root_dir=project_config.dbt_project_path_parent,
             models_dir=Path(dbt_models_dir) if dbt_models_dir else None,
             seeds_dir=Path(dbt_seeds_dir) if dbt_seeds_dir else None,
             snapshots_dir=Path(dbt_snapshots_dir) if dbt_snapshots_dir else None,

cosmos/dbt/graph.py

@@ -112,40 +112,25 @@ def load(
         There is automatic, and manual.
         """
 
-        def load_manual(method: LoadMode) -> None:
-            load_method = {
-                LoadMode.CUSTOM: self.load_via_custom_parser,
-                LoadMode.DBT_LS: self.load_via_dbt_ls,
-                LoadMode.DBT_MANIFEST: self.load_from_dbt_manifest,
-            }
-
-            if method == LoadMode.DBT_MANIFEST:
-                if not self.project.is_manifest_available():
-                    raise CosmosLoadDbtException(f"Unable to load manifest using {self.project.manifest_path}")
-            elif method in [LoadMode.DBT_LS, LoadMode.CUSTOM]:
-                if not self.project.dir:
-                    raise CosmosLoadDbtException(
-                        "Unable to load using dbt_ls or custom method - Project Dir was not provided or does not exist."
-                    )
+        load_method = {
+            LoadMode.CUSTOM: self.load_via_custom_parser,
+            LoadMode.DBT_LS: self.load_via_dbt_ls,
+            LoadMode.DBT_MANIFEST: self.load_from_dbt_manifest,
+        }
 
-            load_method[method]()
-
-        def load_automatic() -> None:
-            try:
-                load_manual(method=LoadMode.DBT_MANIFEST)
-            except CosmosLoadDbtException:
+        if method == LoadMode.AUTOMATIC:
+            if self.project.is_manifest_available():
+                self.load_from_dbt_manifest()
+            else:
                 if execution_mode == ExecutionMode.LOCAL and self.project.is_profile_yml_available():
                     try:
-                        load_manual(method=LoadMode.DBT_LS)
+                        self.load_via_dbt_ls()
                     except FileNotFoundError:
-                        load_manual(method=LoadMode.CUSTOM)
+                        self.load_via_custom_parser()
                 else:
-                    load_manual(method=LoadMode.CUSTOM)
-
-        if method == LoadMode.AUTOMATIC:
-            load_automatic()
+                    self.load_via_custom_parser()
         else:
-            load_manual(method=method)
+            load_method[method]()
 
     def load_via_dbt_ls(self) -> None:
         """
@@ -355,6 +340,10 @@ def load_from_dbt_manifest(self) -> None:
         * self.filtered_nodes
         """
         logger.info("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.name)
+
+        if not self.project.is_manifest_available():
+            raise CosmosLoadDbtException(f"Unable to load manifest using {self.project.manifest_path}")
+
         nodes = {}
         with open(self.project.manifest_path) as fp:  # type: ignore[arg-type]
             manifest = json.load(fp)

dev/dags/basic_cosmos_task_group.py

@@ -38,7 +38,7 @@ def basic_cosmos_task_group() -> None:
     jaffle_shop = DbtTaskGroup(
         group_id="test_123",
         project_config=ProjectConfig(
-            DBT_ROOT_PATH / "jaffle_shop",
+            (DBT_ROOT_PATH / "jaffle_shop").as_posix(),
         ),
         operator_args={"install_deps": True},
         profile_config=profile_config,

dev/dags/cosmos_manifest_example.py

@@ -25,8 +25,7 @@
 cosmos_manifest_example = DbtDag(
     # dbt/cosmos-specific parameters
     project_config=ProjectConfig(
-        DBT_ROOT_PATH / "jaffle_shop",
-        manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json",
+        manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json", project_name="jaffle_shop"
     ),
     profile_config=profile_config,
     render_config=RenderConfig(load_method=LoadMode.DBT_MANIFEST, select=["path:models/customers.sql"]),

tests/test_converter.py

@@ -49,7 +49,8 @@ def test_validate_arguments_tags(argument_key):
 @patch("cosmos.converter.DbtGraph.load")
 def test_converter_creates_dag_with_seed(mock_load_dbt_graph, execution_mode, operator_args):
     """
-    This test will raise exceptions if we are trying to pass incorrect arguments to operator constructors.
+    This test validates that a project, given only a project path as a Path() Object, and seeds
+    is able to successfully generate a converter
     """
     project_config = ProjectConfig(dbt_project_path=SAMPLE_DBT_PROJECT)
     execution_config = ExecutionConfig(execution_mode=execution_mode)
@@ -81,9 +82,9 @@ def test_converter_creates_dag_with_seed(mock_load_dbt_graph, execution_mode, op
 @patch("cosmos.converter.DbtGraph.load")
 def test_converter_creates_dag_with_project_path_str(mock_load_dbt_graph, execution_mode, operator_args):
     """
-    This test will raise exceptions if we are trying to pass incorrect arguments to operator constructors.
+    This test validates that a project, given only a project path as a string, and seeds
+    is able to successfully generate a converter
     """
-
     project_config = ProjectConfig(dbt_project_path=SAMPLE_DBT_PROJECT.as_posix())
     execution_config = ExecutionConfig(execution_mode=execution_mode)
     render_config = RenderConfig(emit_datasets=True)
@@ -114,7 +115,8 @@ def test_converter_creates_dag_with_project_path_str(mock_load_dbt_graph, execut
 @patch("cosmos.converter.DbtGraph.load")
 def test_converter_creates_dag_with_no_project_dir(mock_load_dbt_graph, execution_mode, operator_args):
     """
-    This test will raise exceptions if we are trying to pass incorrect arguments to operator constructors.
+    This test validates that a project, given a manifest path and project name, with seeds
+    is able to successfully generate a converter
     """
     project_config = ProjectConfig(manifest_path=SAMPLE_DBT_MANIFEST.as_posix(), project_name="sample")
     execution_config = ExecutionConfig(execution_mode=execution_mode)