diff --git a/tests/functional/basic/test_project.py b/tests/functional/basic/test_project.py index 2bdb101c913..7a4cb9fd1da 100644 --- a/tests/functional/basic/test_project.py +++ b/tests/functional/basic/test_project.py @@ -4,7 +4,8 @@ import pytest import yaml -from dbt.exceptions import ProjectContractError +from dbt.cli.main import dbtRunner +from dbt.exceptions import DbtProjectError, ProjectContractError from dbt.tests.util import run_dbt, update_config_file, write_config_file simple_model_sql = """ @@ -118,3 +119,51 @@ def test_dbt_cloud_invalid(self, project): with pytest.raises(ProjectContractError) as excinfo: run_dbt() assert expected_err in str(excinfo.value) + + +class TestVersionSpecifierChecksComeBeforeYamlValidation: + def test_version_specifier_checks_before_yaml_validation(self, project) -> None: + runner = dbtRunner() + + # if no version specifier error, we should get a yaml validation error + config_update = {"this-is-not-a-valid-key": "my-value-for-invalid-key"} + update_config_file(config_update, "dbt_project.yml") + result = runner.invoke(["parse"]) + assert result.exception is not None + assert isinstance(result.exception, ProjectContractError) + assert "Additional properties are not allowed" in str(result.exception) + + # add bad version specifier, and assert we get the error for that + update_config_file({"require-dbt-version": [">0.0.0", "<=0.0.1"]}, "dbt_project.yml") + result = runner.invoke(["parse"]) + assert result.exception is not None + assert isinstance(result.exception, DbtProjectError) + assert "This version of dbt is not supported" + + +class TestArchiveNotAllowed: + """At one point in time we supported an 'archive' key in projects, but no longer""" + + def test_archive_not_allowed(self, project): + runner = dbtRunner() + + config_update = { + "archive": { + "source_schema": "a", + "target_schema": "b", + "tables": [ + { + "source_table": "seed", + "target_table": "archive_actual", + "updated_at": "updated_at", + "unique_key": """id || '-' || first_name""", + }, + ], + } + } + update_config_file(config_update, "dbt_project.yml") + + result = runner.invoke(["parse"]) + assert result.exception is not None + assert isinstance(result.exception, ProjectContractError) + assert "Additional properties are not allowed" in str(result.exception) diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index 7d0006570af..ae0ae3928dc 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -2,6 +2,7 @@ import os import unittest from copy import deepcopy +from typing import Any, Dict from unittest import mock import pytest @@ -10,7 +11,7 @@ import dbt.exceptions from dbt.adapters.contracts.connection import DEFAULT_QUERY_COMMENT, QueryComment from dbt.adapters.factory import load_plugin -from dbt.config.project import Project +from dbt.config.project import Project, _get_required_version from dbt.constants import DEPENDENCIES_FILE_NAME from dbt.contracts.project import GitPackage, LocalPackage, PackageConfig from dbt.flags import set_from_args @@ -42,7 +43,7 @@ def test_fixture_paths(self, project: Project): def test__str__(self, project: Project): assert ( str(project) - == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {'database': True, 'schema': True, 'identifier': True}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'vars': {}, 'require-dbt-version': ['=0.0.0'], 'restrict-access': False, 'dbt-cloud': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" + == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'vars': {}, 'require-dbt-version': ['=0.0.0'], 'restrict-access': False, 'dbt-cloud': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" ) def test_get_selector(self, project: Project): @@ -534,3 +535,53 @@ def setUp(self): def test_setting_multiple_flags(self): with pytest.raises(dbt.exceptions.DbtProjectError): set_from_args(self.args, None) + + +class TestGetRequiredVersion: + @pytest.fixture + def project_dict(self) -> Dict[str, Any]: + return { + "name": "test_project", + "require-dbt-version": ">0.0.0", + } + + def test_supported_version(self, project_dict: Dict[str, Any]) -> None: + specifiers = _get_required_version(project_dict=project_dict, verify_version=True) + assert set(x.to_version_string() for x in specifiers) == {">0.0.0"} + + def test_unsupported_version(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = ">99999.0.0" + with pytest.raises( + dbt.exceptions.DbtProjectError, match="This version of dbt is not supported" + ): + _get_required_version(project_dict=project_dict, verify_version=True) + + def test_unsupported_version_no_check(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = ">99999.0.0" + specifiers = _get_required_version(project_dict=project_dict, verify_version=False) + assert set(x.to_version_string() for x in specifiers) == {">99999.0.0"} + + def test_supported_version_range(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = [">0.0.0", "<=99999.0.0"] + specifiers = _get_required_version(project_dict=project_dict, verify_version=True) + assert set(x.to_version_string() for x in specifiers) == {">0.0.0", "<=99999.0.0"} + + def test_unsupported_version_range(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = [">0.0.0", "<=0.0.1"] + with pytest.raises( + dbt.exceptions.DbtProjectError, match="This version of dbt is not supported" + ): + _get_required_version(project_dict=project_dict, verify_version=True) + + def test_unsupported_version_range_no_check(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = [">0.0.0", "<=0.0.1"] + specifiers = _get_required_version(project_dict=project_dict, verify_version=False) + assert set(x.to_version_string() for x in specifiers) == {">0.0.0", "<=0.0.1"} + + def test_impossible_version_range(self, project_dict: Dict[str, Any]) -> None: + project_dict["require-dbt-version"] = [">99999.0.0", "<=0.0.1"] + with pytest.raises( + dbt.exceptions.DbtProjectError, + match="The package version requirement can never be satisfied", + ): + _get_required_version(project_dict=project_dict, verify_version=True) diff --git a/tests/unit/config/test_runtime.py b/tests/unit/config/test_runtime.py index 6d2b18fd896..816ec8f98c3 100644 --- a/tests/unit/config/test_runtime.py +++ b/tests/unit/config/test_runtime.py @@ -1,244 +1,118 @@ import os +import tempfile from argparse import Namespace +from typing import Any, Dict from unittest import mock +import pytest +from pytest_mock import MockerFixture + import dbt.config import dbt.exceptions from dbt import tracking +from dbt.config.profile import Profile +from dbt.config.project import Project +from dbt.config.runtime import RuntimeConfig from dbt.contracts.project import PackageConfig +from dbt.events.types import UnusedResourceConfigPath from dbt.flags import set_from_args from dbt.tests.util import safe_set_invocation_context -from tests.unit.config import ( - BaseConfigTest, - empty_profile_renderer, - project_from_config_norender, - temp_cd, -) - - -class TestRuntimeConfig(BaseConfigTest): - def get_project(self): - return project_from_config_norender( - self.default_project_data, - project_root=self.project_dir, - verify_version=self.args.version_check, +from dbt_common.events.event_manager_client import add_callback_to_manager +from tests.unit.config import BaseConfigTest, temp_cd +from tests.utils import EventCatcher + + +class TestRuntimeConfig: + @pytest.fixture + def args(self) -> Namespace: + return Namespace( + profiles_dir=tempfile.mkdtemp(), + cli_vars={}, + version_check=True, + project_dir=tempfile.mkdtemp(), + target=None, + threads=None, + profile=None, ) - def get_profile(self): - renderer = empty_profile_renderer() - return dbt.config.Profile.from_raw_profiles( - self.default_profile_data, self.default_project_data["profile"], renderer - ) - - def from_parts(self, exc=None): - with self.assertRaisesOrReturns(exc) as err: - project = self.get_project() - profile = self.get_profile() - - result = dbt.config.RuntimeConfig.from_parts(project, profile, self.args) + def test_str(self, profile: Profile, project: Project) -> None: + config = dbt.config.RuntimeConfig.from_parts(project, profile, {}) - if exc is None: - return result - else: - return err + # to make sure nothing terrible happens + str(config) - def test_from_parts(self): - project = self.get_project() - profile = self.get_profile() - config = dbt.config.RuntimeConfig.from_parts(project, profile, self.args) + def test_from_parts(self, args: Namespace, profile: Profile, project: Project): + config = dbt.config.RuntimeConfig.from_parts(project, profile, args) - self.assertEqual(config.cli_vars, {}) - self.assertEqual(config.to_profile_info(), profile.to_profile_info()) + assert config.cli_vars == {} + assert config.to_profile_info() == profile.to_profile_info() # we should have the default quoting set in the full config, but not in # the project # TODO(jeb): Adapters must assert that quoting is populated? expected_project = project.to_project_config() - self.assertEqual(expected_project["quoting"], {}) + assert expected_project["quoting"] == {} expected_project["quoting"] = { "database": True, "identifier": True, "schema": True, } - self.assertEqual(config.to_project_config(), expected_project) - - def test_str(self): - project = self.get_project() - profile = self.get_profile() - config = dbt.config.RuntimeConfig.from_parts(project, profile, {}) - - # to make sure nothing terrible happens - str(config) - - def test_supported_version(self): - self.default_project_data["require-dbt-version"] = ">0.0.0" - conf = self.from_parts() - self.assertEqual(set(x.to_version_string() for x in conf.dbt_version), {">0.0.0"}) - - def test_unsupported_version(self): - self.default_project_data["require-dbt-version"] = ">99999.0.0" - raised = self.from_parts(dbt.exceptions.DbtProjectError) - self.assertIn("This version of dbt is not supported", str(raised.exception)) - - def test_unsupported_version_no_check(self): - self.default_project_data["require-dbt-version"] = ">99999.0.0" - self.args.version_check = False - set_from_args(self.args, None) - conf = self.from_parts() - self.assertEqual(set(x.to_version_string() for x in conf.dbt_version), {">99999.0.0"}) - - def test_supported_version_range(self): - self.default_project_data["require-dbt-version"] = [">0.0.0", "<=99999.0.0"] - conf = self.from_parts() - self.assertEqual( - set(x.to_version_string() for x in conf.dbt_version), {">0.0.0", "<=99999.0.0"} - ) - - def test_unsupported_version_range(self): - self.default_project_data["require-dbt-version"] = [">0.0.0", "<=0.0.1"] - raised = self.from_parts(dbt.exceptions.DbtProjectError) - self.assertIn("This version of dbt is not supported", str(raised.exception)) - - def test_unsupported_version_range_bad_config(self): - self.default_project_data["require-dbt-version"] = [">0.0.0", "<=0.0.1"] - self.default_project_data["some-extra-field-not-allowed"] = True - raised = self.from_parts(dbt.exceptions.DbtProjectError) - self.assertIn("This version of dbt is not supported", str(raised.exception)) - - def test_unsupported_version_range_no_check(self): - self.default_project_data["require-dbt-version"] = [">0.0.0", "<=0.0.1"] - self.args.version_check = False - set_from_args(self.args, None) - conf = self.from_parts() - self.assertEqual( - set(x.to_version_string() for x in conf.dbt_version), {">0.0.0", "<=0.0.1"} - ) - - def test_impossible_version_range(self): - self.default_project_data["require-dbt-version"] = [">99999.0.0", "<=0.0.1"] - raised = self.from_parts(dbt.exceptions.DbtProjectError) - self.assertIn( - "The package version requirement can never be satisfied", str(raised.exception) - ) - - def test_unsupported_version_extra_config(self): - self.default_project_data["some-extra-field-not-allowed"] = True - raised = self.from_parts(dbt.exceptions.DbtProjectError) - self.assertIn("Additional properties are not allowed", str(raised.exception)) - - def test_archive_not_allowed(self): - self.default_project_data["archive"] = [ - { - "source_schema": "a", - "target_schema": "b", - "tables": [ - { - "source_table": "seed", - "target_table": "archive_actual", - "updated_at": "updated_at", - "unique_key": """id || '-' || first_name""", - }, - ], - } - ] - with self.assertRaises(dbt.exceptions.DbtProjectError): - self.get_project() - - def test__warn_for_unused_resource_config_paths_empty(self): - project = self.from_parts() - dbt.flags.WARN_ERROR = True - try: - project.warn_for_unused_resource_config_paths( - { - "models": frozenset( - ( - ("my_test_project", "foo", "bar"), - ("my_test_project", "foo", "baz"), - ) - ) - }, - [], - ) - finally: - dbt.flags.WARN_ERROR = False - - @mock.patch.object(tracking, "active_user") - def test_get_metadata(self, mock_user): - project = self.get_project() - profile = self.get_profile() - config = dbt.config.RuntimeConfig.from_parts(project, profile, self.args) + assert config.to_project_config() == expected_project + def test_get_metadata(self, mocker: MockerFixture, runtime_config: RuntimeConfig) -> None: + mock_user = mocker.patch.object(tracking, "active_user") mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) - metadata = config.get_metadata() + metadata = runtime_config.get_metadata() # ensure user_id and send_anonymous_usage_stats are set correctly - self.assertEqual(metadata.user_id, mock_user.id) - self.assertFalse(metadata.send_anonymous_usage_stats) - - -class TestRuntimeConfigWithConfigs(BaseConfigTest): - def setUp(self): - self.profiles_dir = "/invalid-profiles-path" - self.project_dir = "/invalid-root-path" - super().setUp() - self.default_project_data["project-root"] = self.project_dir - self.default_project_data["models"] = { - "enabled": True, + assert metadata.user_id == mock_user.id + assert not metadata.send_anonymous_usage_stats + + @pytest.fixture + def used_fqns(self) -> Dict[str, Any]: + return {"models": frozenset((("my_test_project", "foo", "bar"),))} + + def test_warn_for_unused_resource_config_paths( + self, + runtime_config: RuntimeConfig, + used_fqns: Dict[str, Any], + ): + catcher = EventCatcher(event_to_catch=UnusedResourceConfigPath) + add_callback_to_manager(catcher.catch) + + runtime_config.models = { "my_test_project": { "foo": { "materialized": "view", "bar": { "materialized": "table", }, - }, - "baz": { - "materialized": "table", - }, - }, - } - self.used = { - "models": frozenset( - ( - ("my_test_project", "foo", "bar"), - ("my_test_project", "foo", "baz"), - ) - ) + "baz": { + "materialized": "table", + }, + } + } } - def get_project(self): - return project_from_config_norender( - self.default_project_data, project_root=self.project_dir, verify_version=True - ) - - def get_profile(self): - renderer = empty_profile_renderer() - return dbt.config.Profile.from_raw_profiles( - self.default_profile_data, self.default_project_data["profile"], renderer - ) - - def from_parts(self, exc=None): - with self.assertRaisesOrReturns(exc) as err: - project = self.get_project() - profile = self.get_profile() + runtime_config.warn_for_unused_resource_config_paths(used_fqns, []) + len(catcher.caught_events) == 1 + expected_msg = "models.my_test_project.foo.baz" + assert expected_msg in str(catcher.caught_events[0].data) - result = dbt.config.RuntimeConfig.from_parts(project, profile, self.args) + def test_warn_for_unused_resource_config_paths_empty_models( + self, + runtime_config: RuntimeConfig, + used_fqns: Dict[str, Any], + ) -> None: + catcher = EventCatcher(event_to_catch=UnusedResourceConfigPath) + add_callback_to_manager(catcher.catch) - if exc is None: - return result - else: - return err + # models should already be empty, but lets ensure it + runtime_config.models = {} - def test__warn_for_unused_resource_config_paths(self): - project = self.from_parts() - with mock.patch("dbt.config.runtime.warn_or_error") as warn_or_error_patch: - project.warn_for_unused_resource_config_paths(self.used, []) - warn_or_error_patch.assert_called_once() - event = warn_or_error_patch.call_args[0][0] - assert type(event).__name__ == "UnusedResourceConfigPath" - msg = event.message() - expected_msg = "- models.my_test_project.baz" - assert expected_msg in msg + runtime_config.warn_for_unused_resource_config_paths(used_fqns, ()) + assert len(catcher.caught_events) == 0 class TestRuntimeConfigFiles(BaseConfigTest): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index f1823fb858f..7c14e8dee5b 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -6,6 +6,7 @@ # All manifest related fixtures. from tests.unit.utils.adapter import * # noqa +from tests.unit.utils.config import * # noqa from tests.unit.utils.event_manager import * # noqa from tests.unit.utils.flags import * # noqa from tests.unit.utils.manifest import * # noqa diff --git a/tests/unit/utils/config.py b/tests/unit/utils/config.py new file mode 100644 index 00000000000..72cb4fa024c --- /dev/null +++ b/tests/unit/utils/config.py @@ -0,0 +1,50 @@ +import pytest + +from dbt.adapters.postgres.connections import PostgresCredentials +from dbt.config.profile import Profile +from dbt.config.project import Project +from dbt.config.renderer import ProfileRenderer +from dbt.config.runtime import RuntimeConfig + + +@pytest.fixture +def credentials() -> PostgresCredentials: + return PostgresCredentials( + database="test_database", + schema="test_schema", + host="test_host", + user="test_user", + port=1337, + password="test_password", + ) + + +@pytest.fixture +def profile() -> Profile: + profile_yaml = { + "target": "postgres", + "outputs": { + "postgres": { + "type": "postgres", + "host": "postgres-db-hostname", + "port": 5555, + "user": "db_user", + "pass": "db_pass", + "dbname": "postgres-db-name", + "schema": "postgres-schema", + "threads": 7, + }, + }, + } + return Profile.from_raw_profile_info( + raw_profile=profile_yaml, profile_name="test_profile", renderer=ProfileRenderer({}) + ) + + +@pytest.fixture +def runtime_config(project: Project, profile: Profile) -> RuntimeConfig: + return RuntimeConfig.from_parts( + project=project, + profile=profile, + args={}, + ) diff --git a/tests/unit/utils/project.py b/tests/unit/utils/project.py index c7215990e6d..2e374b82fac 100644 --- a/tests/unit/utils/project.py +++ b/tests/unit/utils/project.py @@ -45,7 +45,7 @@ def project(selector_config: SelectorConfig) -> Project: log_path="path/to/project/logs", packages_install_path="dbt_packages", packages_specified_path="packages.yml", - quoting={"database": True, "schema": True, "identifier": True}, + quoting={}, models={}, on_run_start=[], on_run_end=[],