diff --git a/cosmos/config.py b/cosmos/config.py index 87baba864..f28574611 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -3,6 +3,7 @@ from __future__ import annotations import contextlib +import shutil import tempfile from dataclasses import InitVar, dataclass, field from pathlib import Path @@ -19,6 +20,14 @@ DEFAULT_PROFILES_FILE_NAME = "profiles.yml" +class CosmosConfigException(Exception): + """ + Exceptions related to user misconfiguration. + """ + + pass + + @dataclass class RenderConfig: """ @@ -51,6 +60,25 @@ class RenderConfig: def __post_init__(self, dbt_project_path: str | Path | None) -> None: self.project_path = Path(dbt_project_path) if dbt_project_path else None + def validate_dbt_command(self, fallback_cmd: str | Path = "") -> None: + """ + Validates that the original dbt command works, if not, attempt to use the fallback_dbt_cmd. + If neither works, raise an exception. + + The fallback behaviour is necessary for Cosmos < 1.2.2 backwards compatibility. + """ + if not shutil.which(self.dbt_executable_path): + if isinstance(fallback_cmd, Path): + fallback_cmd = fallback_cmd.as_posix() + + if fallback_cmd and shutil.which(fallback_cmd): + self.dbt_executable_path = fallback_cmd + else: + raise CosmosConfigException( + "Unable to find the dbt executable, attempted: " + f"<{self.dbt_executable_path}>" + (f" and <{fallback_cmd}>." if fallback_cmd else ".") + ) + class ProjectConfig: """ diff --git a/cosmos/converter.py b/cosmos/converter.py index 45d98a4cf..2f3e1fa2a 100644 --- a/cosmos/converter.py +++ b/cosmos/converter.py @@ -108,6 +108,8 @@ def __init__( if not render_config: render_config = RenderConfig() + render_config.validate_dbt_command(fallback_cmd=execution_config.dbt_executable_path) + # Since we now support both project_config.dbt_project_path, render_config.project_path and execution_config.project_path # We need to ensure that only one interface is being used. if project_config.dbt_project_path and (render_config.project_path or execution_config.project_path): diff --git a/tests/test_config.py b/tests/test_config.py index 9eec48055..d5aadbf7e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,8 +1,9 @@ from pathlib import Path +from unittest.mock import patch import pytest -from cosmos.config import ProfileConfig, ProjectConfig +from cosmos.config import ProfileConfig, ProjectConfig, RenderConfig, CosmosConfigException from cosmos.exceptions import CosmosValueError @@ -121,3 +122,38 @@ def test_profile_config_validate(): profile_config = ProfileConfig(profile_name="test", target_name="test") assert profile_config.validate_profile() is None assert err_info.value.args[0] == "Either profiles_yml_filepath or profile_mapping must be set to render a profile" + + +@patch("cosmos.config.shutil.which", return_value=None) +def test_render_config_without_dbt_cmd(mock_which): + render_config = RenderConfig() + with pytest.raises(CosmosConfigException) as err_info: + render_config.validate_dbt_command("inexistent-dbt") + + error_msg = err_info.value.args[0] + assert error_msg.startswith("Unable to find the dbt executable, attempted: <") + assert error_msg.endswith("dbt> and .") + + +@patch("cosmos.config.shutil.which", return_value=None) +def test_render_config_with_invalid_dbt_commands(mock_which): + render_config = RenderConfig(dbt_executable_path="invalid-dbt") + with pytest.raises(CosmosConfigException) as err_info: + render_config.validate_dbt_command() + + error_msg = err_info.value.args[0] + assert error_msg == "Unable to find the dbt executable, attempted: ." + + +@patch("cosmos.config.shutil.which", side_effect=(None, "fallback-dbt-path")) +def test_render_config_uses_fallback_if_default_not_found(mock_which): + render_config = RenderConfig() + render_config.validate_dbt_command("fallback-dbt-path") + assert render_config.dbt_executable_path == "fallback-dbt-path" + + +@patch("cosmos.config.shutil.which", side_effect=("user-dbt", "fallback-dbt-path")) +def test_render_config_uses_default_if_exists(mock_which): + render_config = RenderConfig(dbt_executable_path="user-dbt") + render_config.validate_dbt_command("fallback-dbt-path") + assert render_config.dbt_executable_path == "user-dbt" diff --git a/tests/test_converter.py b/tests/test_converter.py index 5d89513b3..8ac54f865 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -5,7 +5,7 @@ from cosmos.converter import DbtToAirflowConverter, validate_arguments from cosmos.constants import DbtResourceType, ExecutionMode -from cosmos.config import ProjectConfig, ProfileConfig, ExecutionConfig, RenderConfig +from cosmos.config import ProjectConfig, ProfileConfig, ExecutionConfig, RenderConfig, CosmosConfigException from cosmos.dbt.graph import DbtNode from cosmos.exceptions import CosmosValueError @@ -141,6 +141,40 @@ def test_converter_fails_execution_config_no_project_dir(mock_load_dbt_graph, ex ) +def test_converter_fails_render_config_invalid_dbt_path(): + """ + This test validates that a project, given a manifest path and project name, with seeds + is able to successfully generate a converter + """ + project_config = ProjectConfig(manifest_path=SAMPLE_DBT_MANIFEST.as_posix(), project_name="sample") + execution_config = ExecutionConfig( + execution_mode=ExecutionMode.LOCAL, + dbt_executable_path="invalid-execution-dbt", + dbt_project_path=SAMPLE_DBT_PROJECT, + ) + render_config = RenderConfig( + emit_datasets=True, + dbt_executable_path="invalid-render-dbt", + ) + profile_config = ProfileConfig( + profile_name="my_profile_name", + target_name="my_target_name", + profiles_yml_filepath=SAMPLE_PROFILE_YML, + ) + with pytest.raises(CosmosConfigException) as err_info: + DbtToAirflowConverter( + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + ) + assert ( + err_info.value.args[0] + == "Unable to find the dbt executable, attempted: and ." + ) + + @pytest.mark.parametrize( "execution_mode,operator_args", [