diff --git a/cosmos/config.py b/cosmos/config.py index e052cb9db..9758d0378 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -73,6 +73,10 @@ def parsed_dbt_project_path(self) -> Path | None: def parsed_manifest_path(self) -> Path | None: return Path(self.manifest_path) if self.manifest_path else None + @cached_property + def dbt_project_path_parent(self) -> Path | None: + return self.parsed_dbt_project_path.parent if self.parsed_dbt_project_path else None + def __post_init__(self) -> None: "Converts paths to `Path` objects." if self.parsed_dbt_project_path: diff --git a/cosmos/converter.py b/cosmos/converter.py index ba50ddbb8..ca4f0381f 100644 --- a/cosmos/converter.py +++ b/cosmos/converter.py @@ -106,7 +106,6 @@ def __init__( project_config.validate_project() emit_datasets = render_config.emit_datasets - dbt_root_path = project_config.parsed_dbt_project_path dbt_project_name = project_config.project_name dbt_models_dir = project_config.models_relative_path dbt_seeds_dir = project_config.seeds_relative_path @@ -131,7 +130,7 @@ def __init__( dbt_project = DbtProject( name=dbt_project_name, - root_dir=dbt_root_path.parent if dbt_root_path else None, + root_dir=project_config.dbt_project_path_parent, models_dir=Path(dbt_models_dir) if dbt_models_dir else None, seeds_dir=Path(dbt_seeds_dir) if dbt_seeds_dir else None, snapshots_dir=Path(dbt_snapshots_dir) if dbt_snapshots_dir else None, diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 6bcde06e4..433f469bd 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -112,40 +112,25 @@ def load( There is automatic, and manual. """ - def load_manual(method: LoadMode) -> None: - load_method = { - LoadMode.CUSTOM: self.load_via_custom_parser, - LoadMode.DBT_LS: self.load_via_dbt_ls, - LoadMode.DBT_MANIFEST: self.load_from_dbt_manifest, - } - - if method == LoadMode.DBT_MANIFEST: - if not self.project.is_manifest_available(): - raise CosmosLoadDbtException(f"Unable to load manifest using {self.project.manifest_path}") - elif method in [LoadMode.DBT_LS, LoadMode.CUSTOM]: - if not self.project.dir: - raise CosmosLoadDbtException( - "Unable to load using dbt_ls or custom method - Project Dir was not provided or does not exist." - ) + load_method = { + LoadMode.CUSTOM: self.load_via_custom_parser, + LoadMode.DBT_LS: self.load_via_dbt_ls, + LoadMode.DBT_MANIFEST: self.load_from_dbt_manifest, + } - load_method[method]() - - def load_automatic() -> None: - try: - load_manual(method=LoadMode.DBT_MANIFEST) - except CosmosLoadDbtException: + if method == LoadMode.AUTOMATIC: + if self.project.is_manifest_available(): + self.load_from_dbt_manifest() + else: if execution_mode == ExecutionMode.LOCAL and self.project.is_profile_yml_available(): try: - load_manual(method=LoadMode.DBT_LS) + self.load_via_dbt_ls() except FileNotFoundError: - load_manual(method=LoadMode.CUSTOM) + self.load_via_custom_parser() else: - load_manual(method=LoadMode.CUSTOM) - - if method == LoadMode.AUTOMATIC: - load_automatic() + self.load_via_custom_parser() else: - load_manual(method=method) + load_method[method]() def load_via_dbt_ls(self) -> None: """ @@ -355,6 +340,10 @@ def load_from_dbt_manifest(self) -> None: * self.filtered_nodes """ logger.info("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.name) + + if not self.project.is_manifest_available(): + raise CosmosLoadDbtException(f"Unable to load manifest using {self.project.manifest_path}") + nodes = {} with open(self.project.manifest_path) as fp: # type: ignore[arg-type] manifest = json.load(fp) diff --git a/dev/dags/basic_cosmos_task_group.py b/dev/dags/basic_cosmos_task_group.py index 2f875b8c3..50cb6ed09 100644 --- a/dev/dags/basic_cosmos_task_group.py +++ b/dev/dags/basic_cosmos_task_group.py @@ -38,7 +38,7 @@ def basic_cosmos_task_group() -> None: jaffle_shop = DbtTaskGroup( group_id="test_123", project_config=ProjectConfig( - DBT_ROOT_PATH / "jaffle_shop", + (DBT_ROOT_PATH / "jaffle_shop").as_posix(), ), operator_args={"install_deps": True}, profile_config=profile_config, diff --git a/dev/dags/cosmos_manifest_example.py b/dev/dags/cosmos_manifest_example.py index 14ce42606..2654d0f65 100644 --- a/dev/dags/cosmos_manifest_example.py +++ b/dev/dags/cosmos_manifest_example.py @@ -25,8 +25,7 @@ cosmos_manifest_example = DbtDag( # dbt/cosmos-specific parameters project_config=ProjectConfig( - DBT_ROOT_PATH / "jaffle_shop", - manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json", + manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json", project_name="jaffle_shop" ), profile_config=profile_config, render_config=RenderConfig(load_method=LoadMode.DBT_MANIFEST, select=["path:models/customers.sql"]), diff --git a/tests/test_converter.py b/tests/test_converter.py index f2e427f29..aab4757cb 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -49,7 +49,8 @@ def test_validate_arguments_tags(argument_key): @patch("cosmos.converter.DbtGraph.load") def test_converter_creates_dag_with_seed(mock_load_dbt_graph, execution_mode, operator_args): """ - This test will raise exceptions if we are trying to pass incorrect arguments to operator constructors. + This test validates that a project, given only a project path as a Path() Object, and seeds + is able to successfully generate a converter """ project_config = ProjectConfig(dbt_project_path=SAMPLE_DBT_PROJECT) execution_config = ExecutionConfig(execution_mode=execution_mode) @@ -81,9 +82,9 @@ def test_converter_creates_dag_with_seed(mock_load_dbt_graph, execution_mode, op @patch("cosmos.converter.DbtGraph.load") def test_converter_creates_dag_with_project_path_str(mock_load_dbt_graph, execution_mode, operator_args): """ - This test will raise exceptions if we are trying to pass incorrect arguments to operator constructors. + This test validates that a project, given only a project path as a string, and seeds + is able to successfully generate a converter """ - project_config = ProjectConfig(dbt_project_path=SAMPLE_DBT_PROJECT.as_posix()) execution_config = ExecutionConfig(execution_mode=execution_mode) render_config = RenderConfig(emit_datasets=True) @@ -114,7 +115,8 @@ def test_converter_creates_dag_with_project_path_str(mock_load_dbt_graph, execut @patch("cosmos.converter.DbtGraph.load") def test_converter_creates_dag_with_no_project_dir(mock_load_dbt_graph, execution_mode, operator_args): """ - This test will raise exceptions if we are trying to pass incorrect arguments to operator constructors. + This test validates that a project, given a manifest path and project name, with seeds + is able to successfully generate a converter """ project_config = ProjectConfig(manifest_path=SAMPLE_DBT_MANIFEST.as_posix(), project_name="sample") execution_config = ExecutionConfig(execution_mode=execution_mode)