diff --git a/.changes/unreleased/Under the Hood-20241202-115445.yaml b/.changes/unreleased/Under the Hood-20241202-115445.yaml new file mode 100644 index 00000000000..e6bd65eea49 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241202-115445.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Support upgrading mashumaro to 3.15 +time: 2024-12-02T11:54:45.103325-05:00 +custom: + Author: gshank + Issue: "11044" diff --git a/core/dbt/artifacts/resources/v1/components.py b/core/dbt/artifacts/resources/v1/components.py index 8eb43f35d8e..5d138c9bfde 100644 --- a/core/dbt/artifacts/resources/v1/components.py +++ b/core/dbt/artifacts/resources/v1/components.py @@ -12,7 +12,7 @@ from dbt_common.dataclass_schema import ExtensibleDbtClassMixin, dbtClassMixin from dbt_semantic_interfaces.type_enums import TimeGranularity -NodeVersion = Union[str, float] +NodeVersion = Union[int, float, str] @dataclass diff --git a/core/dbt/artifacts/resources/v1/snapshot.py b/core/dbt/artifacts/resources/v1/snapshot.py index 976c93a4d60..24941c6d54f 100644 --- a/core/dbt/artifacts/resources/v1/snapshot.py +++ b/core/dbt/artifacts/resources/v1/snapshot.py @@ -71,12 +71,6 @@ def final_validate(self): if self.materialized and self.materialized != "snapshot": raise ValidationError("A snapshot must have a materialized value of 'snapshot'") - # Called by "calculate_node_config_dict" in ContextConfigGenerator - def finalize_and_validate(self): - data = self.to_dict(omit_none=True) - self.validate(data) - return self.from_dict(data) - @dataclass class Snapshot(CompiledResource): diff --git a/core/dbt/artifacts/resources/v1/source_definition.py b/core/dbt/artifacts/resources/v1/source_definition.py index 9044307563e..96289b13b42 100644 --- a/core/dbt/artifacts/resources/v1/source_definition.py +++ b/core/dbt/artifacts/resources/v1/source_definition.py @@ -40,7 +40,7 @@ class ExternalTable(AdditionalPropertiesAllowed, Mergeable): file_format: Optional[str] = None row_format: Optional[str] = None tbl_properties: Optional[str] = None - partitions: Optional[Union[List[str], List[ExternalPartition]]] = None + partitions: Optional[Union[List[ExternalPartition], List[str]]] = None def __bool__(self): return self.location is not None diff --git a/core/dbt/context/context_config.py b/core/dbt/context/context_config.py index b1aace475c7..87ef1ab0d04 100644 --- a/core/dbt/context/context_config.py +++ b/core/dbt/context/context_config.py @@ -1,8 +1,9 @@ from abc import abstractmethod from copy import deepcopy from dataclasses import dataclass -from typing import Any, Dict, Generic, Iterator, List, Optional, TypeVar +from typing import Any, Dict, Generic, Iterator, List, Optional, Type, TypeVar +from dbt import hooks from dbt.adapters.factory import get_config_class_by_name from dbt.config import IsFQNResource, Project, RuntimeConfig from dbt.contracts.graph.model_config import get_config_for @@ -26,81 +27,21 @@ class ModelParts(IsFQNResource): C = TypeVar("C", bound=BaseConfig) -class ConfigSource: - def __init__(self, project): - self.project = project +def fix_hooks(config_dict: Dict[str, Any]): + """Given a config dict that may have `pre-hook`/`post-hook` keys, + convert it from the yucky maybe-a-string, maybe-a-dict to a dict. + """ + # Like most of parsing, this is a horrible hack :( + for key in hooks.ModelHookType: + if key in config_dict: + config_dict[key] = [hooks.get_hook_dict(h) for h in config_dict[key]] - def get_config_dict(self, resource_type: NodeType): ... - -class UnrenderedConfig(ConfigSource): - def __init__(self, project: Project): - self.project = project - - def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]: - unrendered = self.project.unrendered.project_dict - if resource_type == NodeType.Seed: - model_configs = unrendered.get("seeds") - elif resource_type == NodeType.Snapshot: - model_configs = unrendered.get("snapshots") - elif resource_type == NodeType.Source: - model_configs = unrendered.get("sources") - elif resource_type == NodeType.Test: - model_configs = unrendered.get("data_tests") - elif resource_type == NodeType.Metric: - model_configs = unrendered.get("metrics") - elif resource_type == NodeType.SemanticModel: - model_configs = unrendered.get("semantic_models") - elif resource_type == NodeType.SavedQuery: - model_configs = unrendered.get("saved_queries") - elif resource_type == NodeType.Exposure: - model_configs = unrendered.get("exposures") - elif resource_type == NodeType.Unit: - model_configs = unrendered.get("unit_tests") - else: - model_configs = unrendered.get("models") - if model_configs is None: - return {} - else: - return model_configs - - -class RenderedConfig(ConfigSource): - def __init__(self, project: Project): - self.project = project - - def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]: - if resource_type == NodeType.Seed: - model_configs = self.project.seeds - elif resource_type == NodeType.Snapshot: - model_configs = self.project.snapshots - elif resource_type == NodeType.Source: - model_configs = self.project.sources - elif resource_type == NodeType.Test: - model_configs = self.project.data_tests - elif resource_type == NodeType.Metric: - model_configs = self.project.metrics - elif resource_type == NodeType.SemanticModel: - model_configs = self.project.semantic_models - elif resource_type == NodeType.SavedQuery: - model_configs = self.project.saved_queries - elif resource_type == NodeType.Exposure: - model_configs = self.project.exposures - elif resource_type == NodeType.Unit: - model_configs = self.project.unit_tests - else: - model_configs = self.project.models - return model_configs - - -class BaseContextConfigGenerator(Generic[T]): +class BaseConfigGenerator(Generic[T]): def __init__(self, active_project: RuntimeConfig): self._active_project = active_project - def get_config_source(self, project: Project) -> ConfigSource: - return RenderedConfig(project) - - def get_node_project(self, project_name: str): + def get_node_project_config(self, project_name: str): if project_name == self._active_project.project_name: return self._active_project dependencies = self._active_project.load_dependencies() @@ -114,9 +55,8 @@ def get_node_project(self, project_name: str): def _project_configs( self, project: Project, fqn: List[str], resource_type: NodeType ) -> Iterator[Dict[str, Any]]: - src = self.get_config_source(project) - model_configs = src.get_config_dict(resource_type) - for level_config in fqn_search(model_configs, fqn): + resource_configs = self.get_resource_configs(project, resource_type) + for level_config in fqn_search(resource_configs, fqn): result = {} for key, value in level_config.items(): if key.startswith("+"): @@ -131,85 +71,112 @@ def _active_project_configs( ) -> Iterator[Dict[str, Any]]: return self._project_configs(self._active_project, fqn, resource_type) - @abstractmethod - def _update_from_config( - self, result: T, partial: Dict[str, Any], validate: bool = False - ) -> T: ... - - @abstractmethod - def initial_result(self, resource_type: NodeType, base: bool) -> T: ... - - def calculate_node_config( + def combine_config_dicts( self, config_call_dict: Dict[str, Any], fqn: List[str], resource_type: NodeType, project_name: str, - base: bool, patch_config_dict: Optional[Dict[str, Any]] = None, - ) -> BaseConfig: - own_config = self.get_node_project(project_name) + ) -> Dict[str, Any]: + """This method takes resource configs from the project, the model (if applicable), + and the patch, and combines them into one config dictionary.""" - result = self.initial_result(resource_type=resource_type, base=base) + project_config = self.get_node_project_config(project_name) + config_cls = get_config_for(resource_type) - project_configs = self._project_configs(own_config, fqn, resource_type) + # creates "default" config object. Unrendered config starts with + # empty dictionary, rendered config starts with to_dict() from empty config object. + config_dict = self.initial_result(config_cls) + + # Update with project configs + project_configs = self._project_configs(project_config, fqn, resource_type) for fqn_config in project_configs: - result = self._update_from_config(result, fqn_config) + config_dict = self._update_from_config(config_cls, config_dict, fqn_config) - # When schema files patch config, it has lower precedence than - # config in the models (config_call_dict), so we add the patch_config_dict - # before the config_call_dict + # Update with schema file configs (patches) if patch_config_dict: - result = self._update_from_config(result, patch_config_dict) + config_dict = self._update_from_config(config_cls, config_dict, patch_config_dict) - # config_calls are created in the 'experimental' model parser and - # the ParseConfigObject (via add_config_call) - result = self._update_from_config(result, config_call_dict) + # Update with config dictionary from sql files (config_call_dict) + config_dict = self._update_from_config(config_cls, config_dict, config_call_dict) - if own_config.project_name != self._active_project.project_name: + # If this is not the root project, update with configs from root project + if project_config.project_name != self._active_project.project_name: for fqn_config in self._active_project_configs(fqn, resource_type): - result = self._update_from_config(result, fqn_config) + config_dict = self._update_from_config(config_cls, config_dict, fqn_config) + + return config_dict + + @abstractmethod + def get_resource_configs( + self, project: Project, resource_type: NodeType + ) -> Dict[str, Any]: ... - # this is mostly impactful in the snapshot config case - # TODO CT-211 - return result # type: ignore[return-value] + @abstractmethod + def _update_from_config( + self, config_cls: Type[BaseConfig], result_dict: Dict[str, Any], partial: Dict[str, Any] + ) -> Dict[str, Any]: ... @abstractmethod - def calculate_node_config_dict( + def initial_result(self, config_cls: Type[BaseConfig]) -> Dict[str, Any]: ... + + @abstractmethod + def generate_node_config( self, config_call_dict: Dict[str, Any], fqn: List[str], resource_type: NodeType, project_name: str, - base: bool, patch_config_dict: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: ... + ): ... -class ContextConfigGenerator(BaseContextConfigGenerator[C]): +class RenderedConfigGenerator(BaseConfigGenerator[C]): + """This class produces the config dictionary used to create the resource config.""" + def __init__(self, active_project: RuntimeConfig): self._active_project = active_project - def get_config_source(self, project: Project) -> ConfigSource: - return RenderedConfig(project) + def get_resource_configs(self, project: Project, resource_type: NodeType) -> Dict[str, Any]: + if resource_type == NodeType.Seed: + resource_configs = project.seeds + elif resource_type == NodeType.Snapshot: + resource_configs = project.snapshots + elif resource_type == NodeType.Source: + resource_configs = project.sources + elif resource_type == NodeType.Test: + resource_configs = project.data_tests + elif resource_type == NodeType.Metric: + resource_configs = project.metrics + elif resource_type == NodeType.SemanticModel: + resource_configs = project.semantic_models + elif resource_type == NodeType.SavedQuery: + resource_configs = project.saved_queries + elif resource_type == NodeType.Exposure: + resource_configs = project.exposures + elif resource_type == NodeType.Unit: + resource_configs = project.unit_tests + else: + resource_configs = project.models + return resource_configs - def initial_result(self, resource_type: NodeType, base: bool) -> C: - # defaults, own_config, config calls, active_config (if != own_config) - config_cls = get_config_for(resource_type, base=base) - # Calculate the defaults. We don't want to validate the defaults, - # because it might be invalid in the case of required config members - # (such as on snapshots!) - result = config_cls.from_dict({}) + def initial_result(self, config_cls: Type[BaseConfig]) -> Dict[str, Any]: + # Produce a dictionary with config defaults. + result = config_cls.from_dict({}).to_dict() return result - def _update_from_config(self, result: C, partial: Dict[str, Any], validate: bool = False) -> C: + def _update_from_config( + self, config_cls: Type[BaseConfig], result_dict: Dict[str, Any], partial: Dict[str, Any] + ) -> Dict[str, Any]: translated = self._active_project.credentials.translate_aliases(partial) translated = self.translate_hook_names(translated) adapter_type = self._active_project.credentials.type adapter_config_cls = get_config_class_by_name(adapter_type) - updated = result.update_from(translated, adapter_config_cls, validate=validate) + # The "update_from" method in BaseConfig merges dictionaries using MergeBehavior + updated = config_cls.update_from(result_dict, translated, adapter_config_cls) return updated def translate_hook_names(self, project_dict): @@ -222,69 +189,107 @@ def translate_hook_names(self, project_dict): project_dict["post-hook"] = project_dict.pop("post_hook") return project_dict - def calculate_node_config_dict( + # RenderedConfigGenerator. Validation is performed, and a config object is returned. + def generate_node_config( self, config_call_dict: Dict[str, Any], fqn: List[str], resource_type: NodeType, project_name: str, - base: bool, patch_config_dict: Optional[dict] = None, - ) -> Dict[str, Any]: - config = self.calculate_node_config( + ) -> BaseConfig: + + config_cls = get_config_for(resource_type) + config_dict = self.combine_config_dicts( config_call_dict=config_call_dict, fqn=fqn, resource_type=resource_type, project_name=project_name, - base=base, patch_config_dict=patch_config_dict, ) + fix_hooks(config_dict) try: - finalized = config.finalize_and_validate() - return finalized.to_dict(omit_none=True) + config_cls.validate(config_dict) + config_obj = config_cls.from_dict(config_dict) + return config_obj except ValidationError as exc: # we got a ValidationError - probably bad types in config() - raise SchemaConfigError(exc, node=config) from exc + config_obj = config_cls.from_dict(config_dict) + raise SchemaConfigError(exc, node=config_obj) from exc -class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]): - def get_config_source(self, project: Project) -> ConfigSource: - return UnrenderedConfig(project) +class UnrenderedConfigGenerator(BaseConfigGenerator[Dict[str, Any]]): + """This class produces the unrendered_config dictionary in the resource.""" - def calculate_node_config_dict( + def get_resource_configs(self, project: Project, resource_type: NodeType) -> Dict[str, Any]: + """Get configs for this resource_type from the project's unrendered config""" + unrendered = project.unrendered.project_dict + if resource_type == NodeType.Seed: + resource_configs = unrendered.get("seeds") + elif resource_type == NodeType.Snapshot: + resource_configs = unrendered.get("snapshots") + elif resource_type == NodeType.Source: + resource_configs = unrendered.get("sources") + elif resource_type == NodeType.Test: + resource_configs = unrendered.get("data_tests") + elif resource_type == NodeType.Metric: + resource_configs = unrendered.get("metrics") + elif resource_type == NodeType.SemanticModel: + resource_configs = unrendered.get("semantic_models") + elif resource_type == NodeType.SavedQuery: + resource_configs = unrendered.get("saved_queries") + elif resource_type == NodeType.Exposure: + resource_configs = unrendered.get("exposures") + elif resource_type == NodeType.Unit: + resource_configs = unrendered.get("unit_tests") + else: + resource_configs = unrendered.get("models") + if resource_configs is None: + return {} + else: + return resource_configs + + # UnrenderedConfigGenerator. No validation is performed and a dictionary is returned. + def generate_node_config( self, config_call_dict: Dict[str, Any], fqn: List[str], resource_type: NodeType, project_name: str, - base: bool, patch_config_dict: Optional[dict] = None, ) -> Dict[str, Any]: - # TODO CT-211 - return self.calculate_node_config( + + result = self.combine_config_dicts( config_call_dict=config_call_dict, fqn=fqn, resource_type=resource_type, project_name=project_name, - base=base, patch_config_dict=patch_config_dict, - ) # type: ignore[return-value] + ) + return result - def initial_result(self, resource_type: NodeType, base: bool) -> Dict[str, Any]: + def initial_result(self, config_cls: Type[BaseConfig]) -> Dict[str, Any]: + # We don't want the config defaults here, just the configs which have + # actually been set. return {} def _update_from_config( self, - result: Dict[str, Any], + config_cls: Type[BaseConfig], + result_dict: Dict[str, Any], partial: Dict[str, Any], - validate: bool = False, ) -> Dict[str, Any]: translated = self._active_project.credentials.translate_aliases(partial) - result.update(translated) - return result + result_dict.update(translated) + return result_dict + +class ConfigBuilder: + """This object is included in various jinja contexts in order to collect the _config_call_dicts + and the _unrendered_config_call dicts from the config calls in sql files. + It is then used to run "build_config_dict" which calls the rendered or unrendered + config generators and returns a config dictionary.""" -class ContextConfig: def __init__( self, active_project: RuntimeConfig, @@ -309,18 +314,14 @@ def add_unrendered_config_call(self, opts: Dict[str, Any]) -> None: def build_config_dict( self, - base: bool = False, - *, rendered: bool = True, patch_config_dict: Optional[dict] = None, ) -> Dict[str, Any]: if rendered: - # TODO CT-211 - src = ContextConfigGenerator(self._active_project) # type: ignore[var-annotated] + config_generator = RenderedConfigGenerator(self._active_project) # type: ignore[var-annotated] config_call_dict = self._config_call_dict - else: - # TODO CT-211 - src = UnrenderedConfigGenerator(self._active_project) # type: ignore[assignment] + else: # unrendered + config_generator = UnrenderedConfigGenerator(self._active_project) # type: ignore[assignment] # preserve legacy behaviour - using unreliable (potentially rendered) _config_call_dict if get_flags().state_modified_compare_more_unrendered_values is False: @@ -333,11 +334,14 @@ def build_config_dict( else: config_call_dict = self._unrendered_config_call_dict - return src.calculate_node_config_dict( + config = config_generator.generate_node_config( config_call_dict=config_call_dict, fqn=self._fqn, resource_type=self._resource_type, project_name=self._project_name, - base=base, patch_config_dict=patch_config_dict, ) + if isinstance(config, BaseConfig): + return config.to_dict(omit_none=True) + else: + return config diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index f9d436a7840..a4717a7418c 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -40,7 +40,7 @@ from dbt.constants import DEFAULT_ENV_PLACEHOLDER from dbt.context.base import Var, contextmember, contextproperty from dbt.context.configured import FQNLookup -from dbt.context.context_config import ContextConfig +from dbt.context.context_config import ConfigBuilder from dbt.context.exceptions_jinja import wrapped_exports from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace from dbt.context.macros import MacroNamespace, MacroNamespaceBuilder @@ -367,14 +367,14 @@ def __call__(self, *args: str) -> MetricReference: class Config(Protocol): - def __init__(self, model, context_config: Optional[ContextConfig]): ... + def __init__(self, model, config_builder: Optional[ConfigBuilder]): ... # Implementation of "config(..)" calls in models class ParseConfigObject(Config): - def __init__(self, model, context_config: Optional[ContextConfig]): + def __init__(self, model, config_builder: Optional[ConfigBuilder]): self.model = model - self.context_config = context_config + self.config_builder = config_builder def _transform_config(self, config): for oldkey in ("pre_hook", "post_hook"): @@ -395,19 +395,19 @@ def __call__(self, *args, **kwargs): opts = self._transform_config(opts) - # it's ok to have a parse context with no context config, but you must + # it's ok to have a parse context with no config builder, but you must # not call it! - if self.context_config is None: - raise DbtRuntimeError("At parse time, did not receive a context config") + if self.config_builder is None: + raise DbtRuntimeError("At parse time, did not receive a config builder") # Track unrendered opts to build parsed node unrendered_config later on if get_flags().state_modified_compare_more_unrendered_values: unrendered_config = statically_parse_unrendered_config(self.model.raw_code) if unrendered_config: - self.context_config.add_unrendered_config_call(unrendered_config) + self.config_builder.add_unrendered_config_call(unrendered_config) - # Use rendered opts to populate context_config - self.context_config.add_config_call(opts) + # Use rendered opts to populate config builder + self.config_builder.add_config_call(opts) return "" def set(self, name, value): @@ -427,7 +427,7 @@ def persist_column_docs(self) -> bool: class RuntimeConfigObject(Config): - def __init__(self, model, context_config: Optional[ContextConfig] = None): + def __init__(self, model, config_builder: Optional[ConfigBuilder] = None): self.model = model # we never use or get a config, only the parser cares @@ -887,7 +887,7 @@ def __init__( config: RuntimeConfig, manifest: Manifest, provider: Provider, - context_config: Optional[ContextConfig], + config_builder: Optional[ConfigBuilder], ) -> None: if provider is None: raise DbtInternalError(f"Invalid provider given to context: {provider}") @@ -896,7 +896,7 @@ def __init__( self.model: Union[Macro, ManifestNode] = model super().__init__(config, manifest, model.package_name) self.sql_results: Dict[str, Optional[AttrDict]] = {} - self.context_config: Optional[ContextConfig] = context_config + self.config_builder: Optional[ConfigBuilder] = config_builder self.provider: Provider = provider self.adapter = get_adapter(self.config) # The macro namespace is used in creating the DatabaseWrapper @@ -1165,7 +1165,7 @@ def ctx_config(self) -> Config: {%- set unique_key = config.require('unique_key') -%} ... """ # noqa - return self.provider.Config(self.model, self.context_config) + return self.provider.Config(self.model, self.config_builder) @contextproperty() def execute(self) -> bool: @@ -1689,12 +1689,12 @@ def generate_parser_model_context( model: ManifestNode, config: RuntimeConfig, manifest: Manifest, - context_config: ContextConfig, + config_builder: ConfigBuilder, ) -> Dict[str, Any]: # The __init__ method of ModelContext also initializes # a ManifestContext object which creates a MacroNamespaceBuilder # which adds every macro in the Manifest. - ctx = ModelContext(model, config, manifest, ParseProvider(), context_config) + ctx = ModelContext(model, config, manifest, ParseProvider(), config_builder) # The 'to_dict' method in ManifestContext moves all of the macro names # in the macro 'namespace' up to top level keys return ctx.to_dict() @@ -1896,14 +1896,14 @@ def __init__( config: RuntimeConfig, manifest: Manifest, provider: Provider, - context_config: Optional[ContextConfig], + config_builder: Optional[ConfigBuilder], macro_resolver: MacroResolver, ) -> None: # this must be before super init so that macro_resolver exists for # build_namespace self.macro_resolver = macro_resolver self.thread_ctx = MacroStack() - super().__init__(model, config, manifest, provider, context_config) + super().__init__(model, config, manifest, provider, config_builder) self._build_test_namespace() # We need to rebuild this because it's already been built by # the ProviderContext with the wrong namespace. @@ -1975,10 +1975,10 @@ def generate_test_context( model: ManifestNode, config: RuntimeConfig, manifest: Manifest, - context_config: ContextConfig, + config_builder: ConfigBuilder, macro_resolver: MacroResolver, ) -> Dict[str, Any]: - ctx = TestContext(model, config, manifest, ParseProvider(), context_config, macro_resolver) + ctx = TestContext(model, config, manifest, ParseProvider(), config_builder, macro_resolver) # The 'to_dict' method in ManifestContext moves all of the macro names # in the macro 'namespace' up to top level keys return ctx.to_dict() diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index b3d5952e268..e18caaaa035 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -53,14 +53,5 @@ class UnitTestNodeConfig(NodeConfig): } -# base resource types are like resource types, except nothing has mandatory -# configs. -BASE_RESOURCE_TYPES: Dict[NodeType, Type[BaseConfig]] = RESOURCE_TYPES.copy() - - -def get_config_for(resource_type: NodeType, base=False) -> Type[BaseConfig]: - if base: - lookup = BASE_RESOURCE_TYPES - else: - lookup = RESOURCE_TYPES - return lookup.get(resource_type, NodeConfig) +def get_config_for(resource_type: NodeType) -> Type[BaseConfig]: + return RESOURCE_TYPES.get(resource_type, NodeConfig) diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index 1d27947a25f..71b54021ddb 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -3,12 +3,12 @@ import os from typing import Any, Dict, Generic, List, Optional, TypeVar -from dbt import hooks, utils +from dbt import utils from dbt.adapters.factory import get_adapter # noqa: F401 from dbt.artifacts.resources import Contract from dbt.clients.jinja import MacroGenerator, get_rendered from dbt.config import RuntimeConfig -from dbt.context.context_config import ContextConfig +from dbt.context.context_config import ConfigBuilder from dbt.context.providers import ( generate_generate_name_macro_context, generate_parser_model_context, @@ -178,15 +178,6 @@ def get_fqn(self, path: str, name: str) -> List[str]: fqn.append(name) return fqn - def _mangle_hooks(self, config): - """Given a config dict that may have `pre-hook`/`post-hook` keys, - convert it from the yucky maybe-a-string, maybe-a-dict to a dict. - """ - # Like most of parsing, this is a horrible hack :( - for key in hooks.ModelHookType: - if key in config: - config[key] = [hooks.get_hook_dict(h) for h in config[key]] - def _create_error_node( self, name: str, path: str, original_file_path: str, raw_code: str, language: str = "sql" ) -> UnparsedNode: @@ -209,7 +200,7 @@ def _create_parsetime_node( self, block: ConfiguredBlockType, path: str, - config: ContextConfig, + config_builder: ConfigBuilder, fqn: List[str], name=None, **kwargs, @@ -239,7 +230,7 @@ def _create_parsetime_node( "raw_code": block.contents, "language": language, "unique_id": self.generate_unique_id(name), - "config": self.config_dict(config), + "config": self.config_dict(config_builder), "checksum": block.file.checksum.to_dict(omit_none=True), } dct.update(kwargs) @@ -257,14 +248,18 @@ def _create_parsetime_node( ) raise DictParseError(exc, node=node) - def _context_for(self, parsed_node: FinalNode, config: ContextConfig) -> Dict[str, Any]: - return generate_parser_model_context(parsed_node, self.root_project, self.manifest, config) + def _context_for( + self, parsed_node: FinalNode, config_builder: ConfigBuilder + ) -> Dict[str, Any]: + return generate_parser_model_context( + parsed_node, self.root_project, self.manifest, config_builder + ) - def render_with_context(self, parsed_node: FinalNode, config: ContextConfig): - # Given the parsed node and a ContextConfig to use during parsing, + def render_with_context(self, parsed_node: FinalNode, config_builder: ConfigBuilder): + # Given the parsed node and a ConfigBuilder to use during parsing, # render the node's sql with macro capture enabled. # Note: this mutates the config object when config calls are rendered. - context = self._context_for(parsed_node, config) + context = self._context_for(parsed_node, config_builder) # this goes through the process of rendering, but just throws away # the rendered result. The "macro capture" is the point? @@ -274,14 +269,12 @@ def render_with_context(self, parsed_node: FinalNode, config: ContextConfig): # This is taking the original config for the node, converting it to a dict, # updating the config with new config passed in, then re-creating the # config from the dict in the node. - def update_parsed_node_config_dict( + def clean_and_fix_config_dict( self, parsed_node: FinalNode, config_dict: Dict[str, Any] ) -> None: # Overwrite node config final_config_dict = parsed_node.config.to_dict(omit_none=True) final_config_dict.update({k.strip(): v for (k, v) in config_dict.items()}) - # re-mangle hooks, in case we got new ones - self._mangle_hooks(final_config_dict) parsed_node.config = parsed_node.config.from_dict(final_config_dict) def update_parsed_node_relation_names( @@ -308,29 +301,33 @@ def update_parsed_node_relation_names( def update_parsed_node_config( self, parsed_node: FinalNode, - config: ContextConfig, + config_builder: ConfigBuilder, context=None, patch_config_dict=None, patch_file_id=None, ) -> None: - """Given the ContextConfig used for parsing and the parsed node, - generate and set the true values to use, overriding the temporary parse - values set in _build_intermediate_parsed_node. + """Given the ConfigBuilder used for parsing and the parsed node, + generate the final resource config and the unrendered_config """ - # build_config_dict takes the config_call_dict in the ContextConfig object - # and calls calculate_node_config to combine dbt_project configs and - # config calls from SQL files, plus patch configs (from schema files) - # This normalize the config for a model node due #8520; should be improved latter if not patch_config_dict: patch_config_dict = {} if ( parsed_node.resource_type == NodeType.Model and parsed_node.language == ModelLanguage.python ): + # This normalize the config for a python model node due #8520; should be improved latter if "materialized" not in patch_config_dict: patch_config_dict["materialized"] = "table" - config_dict = config.build_config_dict(patch_config_dict=patch_config_dict) + + # build_config_dict takes the config_call_dict in the ConfigBuilder object + # and calls generate_node_config to combine dbt_project configs and + # config calls from SQL files, plus patch configs (from schema files). + # Validation is performed when building the rendered config_dict and + # hooks are converted into hook objects for later rendering. + config_dict = config_builder.build_config_dict( + rendered=True, patch_config_dict=patch_config_dict + ) # Set tags on node provided in config blocks. Tags are additive, so even if # config has been built before, we don't have to reset tags in the parsed_node. @@ -396,16 +393,16 @@ def update_parsed_node_config( # unrendered_config is used to compare the original database/schema/alias # values and to handle 'same_config' and 'same_contents' calls - parsed_node.unrendered_config = config.build_config_dict( + parsed_node.unrendered_config = config_builder.build_config_dict( rendered=False, patch_config_dict=patch_config_dict ) - parsed_node.config_call_dict = config._config_call_dict - parsed_node.unrendered_config_call_dict = config._unrendered_config_call_dict + parsed_node.config_call_dict = config_builder._config_call_dict + parsed_node.unrendered_config_call_dict = config_builder._unrendered_config_call_dict # do this once before we parse the node database/schema/alias, so # parsed_node.config is what it would be if they did nothing - self.update_parsed_node_config_dict(parsed_node, config_dict) + self.clean_and_fix_config_dict(parsed_node, config_dict) # This updates the node database/schema/alias/relation_name self.update_parsed_node_relation_names(parsed_node, config_dict) @@ -413,44 +410,36 @@ def update_parsed_node_config( if parsed_node.resource_type == NodeType.Test: return - # at this point, we've collected our hooks. Use the node context to - # render each hook and collect refs/sources + # Use the node context to render each hook and collect refs/sources. assert hasattr(parsed_node.config, "pre_hook") and hasattr(parsed_node.config, "post_hook") hooks = list(itertools.chain(parsed_node.config.pre_hook, parsed_node.config.post_hook)) # skip context rebuilding if there aren't any hooks if not hooks: return if not context: - context = self._context_for(parsed_node, config) + context = self._context_for(parsed_node, config_builder) for hook in hooks: get_rendered(hook.sql, context, parsed_node, capture_macros=True) - def initial_config(self, fqn: List[str]) -> ContextConfig: - config_version = min([self.project.config_version, self.root_project.config_version]) - if config_version == 2: - return ContextConfig( - self.root_project, - fqn, - self.resource_type, - self.project.project_name, - ) - else: - raise DbtInternalError( - f"Got an unexpected project version={config_version}, expected 2" - ) + def initial_config_builder(self, fqn: List[str]) -> ConfigBuilder: + return ConfigBuilder( + self.root_project, + fqn, + self.resource_type, + self.project.project_name, + ) def config_dict( self, - config: ContextConfig, + config_builder: ConfigBuilder, ) -> Dict[str, Any]: - config_dict = config.build_config_dict(base=True) - self._mangle_hooks(config_dict) + config_dict = config_builder.build_config_dict(rendered=True) return config_dict - def render_update(self, node: FinalNode, config: ContextConfig) -> None: + def render_update(self, node: FinalNode, config_builder: ConfigBuilder) -> None: try: - context = self.render_with_context(node, config) - self.update_parsed_node_config(node, config, context=context) + context = self.render_with_context(node, config_builder) + self.update_parsed_node_config(node, config_builder, context=context) except ValidationError as exc: # we got a ValidationError - probably bad types in config() raise ConfigUpdateError(exc, node=node) from exc @@ -465,15 +454,15 @@ def parse_node(self, block: ConfiguredBlockType) -> FinalNode: compiled_path: str = self.get_compiled_path(block) fqn = self.get_fqn(compiled_path, block.name) - config: ContextConfig = self.initial_config(fqn) + config_builder: ConfigBuilder = self.initial_config_builder(fqn) node = self._create_parsetime_node( block=block, path=compiled_path, - config=config, + config_builder=config_builder, fqn=fqn, ) - self.render_update(node, config) + self.render_update(node, config_builder) self.add_result_node(block, node) return node diff --git a/core/dbt/parser/hooks.py b/core/dbt/parser/hooks.py index bcc25c0d937..f7836b7e57e 100644 --- a/core/dbt/parser/hooks.py +++ b/core/dbt/parser/hooks.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Iterable, Iterator, List, Tuple, Union -from dbt.context.context_config import ContextConfig +from dbt.context.context_config import ConfigBuilder from dbt.contracts.files import FilePath from dbt.contracts.graph.nodes import HookNode from dbt.node_types import NodeType, RunHookType @@ -92,7 +92,7 @@ def _create_parsetime_node( self, block: HookBlock, path: str, - config: ContextConfig, + config_builder: ConfigBuilder, fqn: List[str], name=None, **kwargs, @@ -101,7 +101,7 @@ def _create_parsetime_node( return super()._create_parsetime_node( block=block, path=path, - config=config, + config_builder=config_builder, fqn=fqn, index=block.index, name=name, diff --git a/core/dbt/parser/models.py b/core/dbt/parser/models.py index 06e11a89649..5084ee03842 100644 --- a/core/dbt/parser/models.py +++ b/core/dbt/parser/models.py @@ -10,7 +10,7 @@ from dbt import utils from dbt.artifacts.resources import RefArgs from dbt.clients.jinja import get_rendered -from dbt.context.context_config import ContextConfig +from dbt.context.context_config import ConfigBuilder from dbt.contracts.graph.nodes import ModelNode from dbt.exceptions import ( ModelConfigError, @@ -233,15 +233,15 @@ def parse_python_model(self, node, config, context): config_keys_defaults=config_keys_defaults, ) - def render_update(self, node: ModelNode, config: ContextConfig) -> None: + def render_update(self, node: ModelNode, config_builder: ConfigBuilder) -> None: self.manifest._parsing_info.static_analysis_path_count += 1 flags = get_flags() if node.language == ModelLanguage.python: try: verify_python_model_code(node) - context = self._context_for(node, config) - self.parse_python_model(node, config, context) - self.update_parsed_node_config(node, config, context=context) + context = self._context_for(node, config_builder) + self.parse_python_model(node, config_builder, context) + self.update_parsed_node_config(node, config_builder, context=context) except ValidationError as exc: # we got a ValidationError - probably bad types in config() @@ -250,7 +250,7 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None: elif not flags.STATIC_PARSER: # jinja rendering - super().render_update(node, config) + super().render_update(node, config_builder) return # only sample for experimental parser correctness on normal runs, @@ -277,9 +277,9 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None: statically_parsed: Optional[Union[str, Dict[str, List[Any]]]] = None experimental_sample: Optional[Union[str, Dict[str, List[Any]]]] = None exp_sample_node: Optional[ModelNode] = None - exp_sample_config: Optional[ContextConfig] = None + exp_sample_config_builder: Optional[ConfigBuilder] = None jinja_sample_node: Optional[ModelNode] = None - jinja_sample_config: Optional[ContextConfig] = None + jinja_sample_config_builder: Optional[ConfigBuilder] = None result: List[str] = [] # sample the experimental parser only during a normal run @@ -295,8 +295,10 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None: if isinstance(experimental_sample, dict): model_parser_copy = self.partial_deepcopy() exp_sample_node = deepcopy(node) - exp_sample_config = deepcopy(config) - model_parser_copy.populate(exp_sample_node, exp_sample_config, experimental_sample) + exp_sample_config_builder = deepcopy(config_builder) + model_parser_copy.populate( + exp_sample_node, exp_sample_config_builder, experimental_sample + ) # use the experimental parser exclusively if the flag is on if flags.USE_EXPERIMENTAL_PARSER: statically_parsed = self.run_experimental_parser(node) @@ -317,36 +319,36 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None: # but we can't really guarantee that going forward. model_parser_copy = self.partial_deepcopy() jinja_sample_node = deepcopy(node) - jinja_sample_config = deepcopy(config) + jinja_sample_config_builder = deepcopy(config_builder) # rendering mutates the node and the config super(ModelParser, model_parser_copy).render_update( - jinja_sample_node, jinja_sample_config + jinja_sample_node, jinja_sample_config_builder ) # update the unrendered config with values from the static parser. # values from yaml files are in there already - self.populate(node, config, statically_parsed) + self.populate(node, config_builder, statically_parsed) # if we took a jinja sample, compare now that the base node has been populated - if jinja_sample_node is not None and jinja_sample_config is not None: + if jinja_sample_node is not None and jinja_sample_config_builder is not None: result = _get_stable_sample_result( - jinja_sample_node, jinja_sample_config, node, config + jinja_sample_node, jinja_sample_config_builder, node, config_builder ) # if we took an experimental sample, compare now that the base node has been populated - if exp_sample_node is not None and exp_sample_config is not None: + if exp_sample_node is not None and exp_sample_config_builder is not None: result = _get_exp_sample_result( exp_sample_node, - exp_sample_config, + exp_sample_config_builder, node, - config, + config_builder, ) self.manifest._parsing_info.static_analysis_parsed_path_count += 1 # if the static parser didn't succeed, fall back to jinja else: # jinja rendering - super().render_update(node, config) + super().render_update(node, config_builder) # if sampling, add the correct messages for tracking if exp_sample and isinstance(experimental_sample, str): @@ -432,13 +434,15 @@ def _has_banned_macro(self, node: ModelNode) -> bool: # this method updates the model node rendered and unrendered config as well # as the node object. Used to populate these values when circumventing jinja # rendering like the static parser. - def populate(self, node: ModelNode, config: ContextConfig, statically_parsed: Dict[str, Any]): + def populate( + self, node: ModelNode, config_builder: ConfigBuilder, statically_parsed: Dict[str, Any] + ): # manually fit configs in - config._config_call_dict = _get_config_call_dict(statically_parsed) + config_builder._config_call_dict = _get_config_call_dict(statically_parsed) # if there are hooks present this, it WILL render jinja. Will need to change # when the experimental parser supports hooks - self.update_parsed_node_config(node, config) + self.update_parsed_node_config(node, config_builder) # update the unrendered config with values from the file. # values from yaml files are in there already @@ -488,11 +492,13 @@ def _shift_sources(static_parser_result: Dict[str, List[Any]]) -> Dict[str, List # returns a list of string codes to be sent as a tracking event def _get_exp_sample_result( sample_node: ModelNode, - sample_config: ContextConfig, + sample_config_builder: ConfigBuilder, node: ModelNode, - config: ContextConfig, + config_builder: ConfigBuilder, ) -> List[str]: - result: List[Tuple[int, str]] = _get_sample_result(sample_node, sample_config, node, config) + result: List[Tuple[int, str]] = _get_sample_result( + sample_node, sample_config_builder, node, config_builder + ) def process(codemsg): code, msg = codemsg @@ -504,11 +510,13 @@ def process(codemsg): # returns a list of string codes to be sent as a tracking event def _get_stable_sample_result( sample_node: ModelNode, - sample_config: ContextConfig, + sample_config_builder: ConfigBuilder, node: ModelNode, - config: ContextConfig, + config_builder: ConfigBuilder, ) -> List[str]: - result: List[Tuple[int, str]] = _get_sample_result(sample_node, sample_config, node, config) + result: List[Tuple[int, str]] = _get_sample_result( + sample_node, sample_config_builder, node, config_builder + ) def process(codemsg): code, msg = codemsg @@ -521,20 +529,20 @@ def process(codemsg): # before being sent as a tracking event def _get_sample_result( sample_node: ModelNode, - sample_config: ContextConfig, + sample_config_builder: ConfigBuilder, node: ModelNode, - config: ContextConfig, + config_builder: ConfigBuilder, ) -> List[Tuple[int, str]]: result: List[Tuple[int, str]] = [] # look for false positive configs - for k in sample_config._config_call_dict.keys(): - if k not in config._config_call_dict.keys(): + for k in sample_config_builder._config_call_dict.keys(): + if k not in config_builder._config_call_dict.keys(): result += [(2, "false_positive_config_value")] break # look for missed configs - for k in config._config_call_dict.keys(): - if k not in sample_config._config_call_dict.keys(): + for k in config_builder._config_call_dict.keys(): + if k not in sample_config_builder._config_call_dict.keys(): result += [(3, "missed_config_value")] break diff --git a/core/dbt/parser/schema_generic_tests.py b/core/dbt/parser/schema_generic_tests.py index 58be6dc94be..bdd7228a907 100644 --- a/core/dbt/parser/schema_generic_tests.py +++ b/core/dbt/parser/schema_generic_tests.py @@ -7,7 +7,7 @@ from dbt.artifacts.resources import NodeVersion, RefArgs from dbt.clients.jinja import add_rendered_test_kwargs, get_rendered from dbt.context.configured import SchemaYamlVars, generate_schema_yml_context -from dbt.context.context_config import ContextConfig +from dbt.context.context_config import ConfigBuilder from dbt.context.macro_resolver import MacroResolver from dbt.context.providers import generate_test_context from dbt.contracts.files import FileHash @@ -88,7 +88,7 @@ def create_test_node( self, target: Union[UnpatchedSourceDefinition, UnparsedNodeUpdate], path: str, - config: ContextConfig, + config_builder: ConfigBuilder, tags: List[str], fqn: List[str], name: str, @@ -130,7 +130,7 @@ def get_hashable_md(data: Union[str, int, float, List, Dict]) -> Union[str, List "raw_code": raw_code, "language": "sql", "unique_id": self.generate_unique_id(name, test_hash), - "config": self.config_dict(config), + "config": self.config_dict(config_builder), "test_metadata": test_metadata, "column_name": column_name, "checksum": FileHash.empty().to_dict(omit_none=True), @@ -200,11 +200,11 @@ def parse_generic_test( relative_path = str(path.relative_to(*path.parts[:1])) fqn = self.get_fqn(relative_path, builder.fqn_name) - # this is the ContextConfig that is used in render_update - config: ContextConfig = self.initial_config(fqn) - # Adding the builder's config to the ContextConfig + # this is the ConfigBuilder that is used in render_update + config_builder: ConfigBuilder = self.initial_config_builder(fqn) + # Adding the builder's config to the ConfigBuilder # is needed to ensure the config makes it to the pre_model hook which dbt-snowflake needs - config.add_config_call(builder.config) + config_builder.add_config_call(builder.config) # builder.args contains keyword args for the test macro, # not configs which have been separated out in the builder. # The keyword args are not completely rendered until compilation. @@ -223,7 +223,7 @@ def parse_generic_test( node = self.create_test_node( target=target, path=compiled_path, - config=config, + config_builder=config_builder, fqn=fqn, tags=tags, name=builder.fqn_name, @@ -233,7 +233,7 @@ def parse_generic_test( file_key_name=file_key_name, description=builder.description, ) - self.render_test_update(node, config, builder, schema_file_id) + self.render_test_update(node, config_builder, builder, schema_file_id) return node @@ -278,7 +278,7 @@ def store_env_vars(self, target, schema_file_id, env_vars): # In the future we will look at generalizing this # more to handle additional macros or to use static # parsing to avoid jinja overhead. - def render_test_update(self, node, config, builder, schema_file_id): + def render_test_update(self, node, config_builder, builder, schema_file_id): macro_unique_id = self.macro_resolver.get_macro_id( node.package_name, "test_" + builder.name ) @@ -287,9 +287,9 @@ def render_test_update(self, node, config, builder, schema_file_id): node.depends_on.add_macro(macro_unique_id) if macro_unique_id in ["macro.dbt.test_not_null", "macro.dbt.test_unique"]: config_call_dict = builder.get_static_config() - config._config_call_dict = config_call_dict + config_builder._config_call_dict = config_call_dict # This sets the config from dbt_project - self.update_parsed_node_config(node, config) + self.update_parsed_node_config(node, config_builder) # source node tests are processed at patch_source time if isinstance(builder.target, UnpatchedSourceDefinition): sources = [builder.target.fqn[-2], builder.target.fqn[-1]] @@ -303,7 +303,7 @@ def render_test_update(self, node, config, builder, schema_file_id): node, self.root_project, self.manifest, - config, + config_builder, self.macro_resolver, ) # update with rendered test kwargs (which collects any refs) @@ -312,7 +312,7 @@ def render_test_update(self, node, config, builder, schema_file_id): add_rendered_test_kwargs(context, node, capture_macros=True) # the parsed node is not rendered in the native context. get_rendered(node.raw_code, context, node, capture_macros=True) - self.update_parsed_node_config(node, config) + self.update_parsed_node_config(node, config_builder) # env_vars should have been updated in the context env_var method except ValidationError as exc: # we got a ValidationError - probably bad types in config() @@ -351,14 +351,14 @@ def add_test_node(self, block: GenericTestBlock, node: GenericTestNode): def render_with_context( self, node: GenericTestNode, - config: ContextConfig, + config_builder: ConfigBuilder, ) -> None: - """Given the parsed node and a ContextConfig to use during + """Given the parsed node and a ConfigBuilder to use during parsing, collect all the refs that might be squirreled away in the test arguments. This includes the implicit "model" argument. """ # make a base context that doesn't have the magic kwargs field - context = self._context_for(node, config) + context = self._context_for(node, config_builder) # update it with the rendered test kwargs (which collects any refs) add_rendered_test_kwargs(context, node, capture_macros=True) diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index aca239db153..04bdfde33ad 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -23,8 +23,8 @@ ) from dbt.clients.jinja import get_rendered from dbt.context.context_config import ( - BaseContextConfigGenerator, - ContextConfigGenerator, + BaseConfigGenerator, + RenderedConfigGenerator, UnrenderedConfigGenerator, ) from dbt.context.providers import ( @@ -94,6 +94,7 @@ def parse_exposure(self, unparsed: UnparsedExposure) -> None: fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) + # Also validates config = self._generate_exposure_config( target=unparsed, fqn=fqn, @@ -101,8 +102,6 @@ def parse_exposure(self, unparsed: UnparsedExposure) -> None: rendered=True, ) - config = config.finalize_and_validate() - unrendered_config = self._generate_exposure_config( target=unparsed, fqn=fqn, @@ -153,9 +152,9 @@ def parse_exposure(self, unparsed: UnparsedExposure) -> None: def _generate_exposure_config( self, target: UnparsedExposure, fqn: List[str], package_name: str, rendered: bool ): - generator: BaseContextConfigGenerator + generator: BaseConfigGenerator if rendered: - generator = ContextConfigGenerator(self.root_project) + generator = RenderedConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) @@ -164,12 +163,11 @@ def _generate_exposure_config( # apply exposure configs precedence_configs.update(target.config) - return generator.calculate_node_config( + return generator.generate_node_config( config_call_dict={}, fqn=fqn, resource_type=NodeType.Exposure, project_name=package_name, - base=False, patch_config_dict=precedence_configs, ) @@ -382,6 +380,7 @@ def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] = fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) + # Also validates config = self._generate_metric_config( target=unparsed, fqn=fqn, @@ -389,8 +388,6 @@ def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] = rendered=True, ) - config = config.finalize_and_validate() - unrendered_config = self._generate_metric_config( target=unparsed, fqn=fqn, @@ -439,23 +436,22 @@ def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] = def _generate_metric_config( self, target: UnparsedMetric, fqn: List[str], package_name: str, rendered: bool ): - generator: BaseContextConfigGenerator + generator: BaseConfigGenerator if rendered: - generator = ContextConfigGenerator(self.root_project) + generator = RenderedConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) - # configs with precendence set + # configs with precedence set precedence_configs = dict() # first apply metric configs precedence_configs.update(target.config) - config = generator.calculate_node_config( + config = generator.generate_node_config( config_call_dict={}, fqn=fqn, resource_type=NodeType.Metric, project_name=package_name, - base=False, patch_config_dict=precedence_configs, ) return config @@ -608,9 +604,9 @@ def _create_metric( def _generate_semantic_model_config( self, target: UnparsedSemanticModel, fqn: List[str], package_name: str, rendered: bool ): - generator: BaseContextConfigGenerator + generator: BaseConfigGenerator if rendered: - generator = ContextConfigGenerator(self.root_project) + generator = RenderedConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) @@ -619,12 +615,11 @@ def _generate_semantic_model_config( # first apply semantic model configs precedence_configs.update(target.config) - config = generator.calculate_node_config( + config = generator.generate_node_config( config_call_dict={}, fqn=fqn, resource_type=NodeType.SemanticModel, project_name=package_name, - base=False, patch_config_dict=precedence_configs, ) @@ -638,6 +633,7 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel) -> None: fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) + # Also validates config = self._generate_semantic_model_config( target=unparsed, fqn=fqn, @@ -645,8 +641,6 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel) -> None: rendered=True, ) - config = config.finalize_and_validate() - unrendered_config = self._generate_semantic_model_config( target=unparsed, fqn=fqn, @@ -723,9 +717,9 @@ def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: def _generate_saved_query_config( self, target: UnparsedSavedQuery, fqn: List[str], package_name: str, rendered: bool ): - generator: BaseContextConfigGenerator + generator: BaseConfigGenerator if rendered: - generator = ContextConfigGenerator(self.root_project) + generator = RenderedConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) @@ -734,12 +728,11 @@ def _generate_saved_query_config( # first apply semantic model configs precedence_configs.update(target.config) - config = generator.calculate_node_config( + config = generator.generate_node_config( config_call_dict={}, fqn=fqn, resource_type=NodeType.SavedQuery, project_name=package_name, - base=False, patch_config_dict=precedence_configs, ) @@ -783,6 +776,7 @@ def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None: fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) + # Also validates config = self._generate_saved_query_config( target=unparsed, fqn=fqn, @@ -790,8 +784,6 @@ def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None: rendered=True, ) - config = config.finalize_and_validate() - unrendered_config = self._generate_saved_query_config( target=unparsed, fqn=fqn, diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index e9c66e184e4..773108ea57b 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -11,7 +11,7 @@ from dbt.clients.yaml_helper import load_yaml_text from dbt.config import RuntimeConfig from dbt.context.configured import SchemaYamlVars, generate_schema_yml_context -from dbt.context.context_config import ContextConfig +from dbt.context.context_config import ConfigBuilder from dbt.contracts.files import SchemaSourceFile, SourceFile from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ( @@ -299,7 +299,7 @@ def _add_yaml_snapshot_nodes_to_manifest( snapshot_node = parser._create_parsetime_node( block, compiled_path, - parser.initial_config(fqn), + parser.initial_config_builder(fqn), fqn, snapshot["name"], ) @@ -694,20 +694,20 @@ def patch_node_config(self, node, patch) -> None: unique_id=node.unique_id, field_value=patch.config["access"], ) - # Get the ContextConfig that's used in calculating the config + # Get the ConfigBuilder that's used in calculating the config # This must match the model resource_type that's being patched - config = ContextConfig( + config_builder = ConfigBuilder( self.schema_parser.root_project, node.fqn, node.resource_type, self.schema_parser.project.project_name, ) # We need to re-apply the config_call_dict after the patch config - config._config_call_dict = node.config_call_dict - config._unrendered_config_call_dict = node.unrendered_config_call_dict + config_builder._config_call_dict = node.config_call_dict + config_builder._unrendered_config_call_dict = node.unrendered_config_call_dict self.schema_parser.update_parsed_node_config( node, - config, + config_builder, patch_config_dict=patch.config, patch_file_id=patch.file_id, ) diff --git a/core/dbt/parser/seeds.py b/core/dbt/parser/seeds.py index 23c77e1ed7c..cebaaa88f4c 100644 --- a/core/dbt/parser/seeds.py +++ b/core/dbt/parser/seeds.py @@ -1,4 +1,4 @@ -from dbt.context.context_config import ContextConfig +from dbt.context.context_config import ConfigBuilder from dbt.contracts.graph.nodes import SeedNode from dbt.node_types import NodeType from dbt.parser.base import SimpleSQLParser @@ -24,5 +24,5 @@ def resource_type(self) -> NodeType: def get_compiled_path(cls, block: FileBlock): return block.path.relative_path - def render_with_context(self, parsed_node: SeedNode, config: ContextConfig) -> None: + def render_with_context(self, parsed_node: SeedNode, config_builder: ConfigBuilder) -> None: """Seeds don't need to do any rendering.""" diff --git a/core/dbt/parser/sources.py b/core/dbt/parser/sources.py index 0fe882750ae..0545ce9ee80 100644 --- a/core/dbt/parser/sources.py +++ b/core/dbt/parser/sources.py @@ -8,8 +8,8 @@ from dbt.artifacts.resources import FreshnessThreshold, SourceConfig, Time from dbt.config import RuntimeConfig from dbt.context.context_config import ( - BaseContextConfigGenerator, - ContextConfigGenerator, + BaseConfigGenerator, + RenderedConfigGenerator, UnrenderedConfigGenerator, ) from dbt.contracts.graph.manifest import Manifest, SourceKey @@ -146,13 +146,12 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> SourceDefinition: # make sure we don't do duplicate tags from source + table tags = sorted(set(itertools.chain(source.tags, table.tags))) + # This will also validate config = self._generate_source_config( target=target, rendered=True, ) - config = config.finalize_and_validate() - unrendered_config = self._generate_source_config( target=target, rendered=False, @@ -287,9 +286,9 @@ def parse_source_test( return node def _generate_source_config(self, target: UnpatchedSourceDefinition, rendered: bool): - generator: BaseContextConfigGenerator + generator: BaseConfigGenerator if rendered: - generator = ContextConfigGenerator(self.root_project) + generator = RenderedConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) @@ -302,12 +301,11 @@ def _generate_source_config(self, target: UnpatchedSourceDefinition, rendered: b # it works while source configs can only include `enabled`. precedence_configs.update(target.table.config) - return generator.calculate_node_config( + return generator.generate_node_config( config_call_dict={}, fqn=target.fqn, resource_type=NodeType.Source, project_name=target.package_name, - base=False, patch_config_dict=precedence_configs, ) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 38a9c81fb3d..d17bc72271a 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -9,7 +9,7 @@ from dbt import utils from dbt.artifacts.resources import ModelConfig, UnitTestConfig, UnitTestFormat from dbt.config import RuntimeConfig -from dbt.context.context_config import ContextConfig +from dbt.context.context_config import ConfigBuilder from dbt.context.providers import generate_parse_exposure, get_rendered from dbt.contracts.files import FileHash, SchemaSourceFile from dbt.contracts.graph.manifest import Manifest @@ -314,13 +314,15 @@ def _get_unit_test(self, data: Dict[str, Any]) -> UnparsedUnitTest: def _build_unit_test_config( self, unit_test_fqn: List[str], config_dict: Dict[str, Any] ) -> UnitTestConfig: - config = ContextConfig( + config_builder = ConfigBuilder( self.schema_parser.root_project, unit_test_fqn, NodeType.Unit, self.schema_parser.project.project_name, ) - unit_test_config_dict = config.build_config_dict(patch_config_dict=config_dict) + unit_test_config_dict = config_builder.build_config_dict( + rendered=True, patch_config_dict=config_dict + ) unit_test_config_dict = self.render_entry(unit_test_config_dict) return UnitTestConfig.from_dict(unit_test_config_dict) diff --git a/core/setup.py b/core/setup.py index be77d1ba73b..a78e64c6e6d 100644 --- a/core/setup.py +++ b/core/setup.py @@ -51,7 +51,7 @@ # Pin to the patch or minor version, and bump in each new minor version of dbt-core. "agate>=1.7.0,<1.10", "Jinja2>=3.1.3,<4", - "mashumaro[msgpack]>=3.9,<3.15", + "mashumaro[msgpack]>=3.15,<4.0", # ---- # dbt-core uses these packages in standard ways. Pin to the major version, and check compatibility # with major versions in each new minor version of dbt-core. diff --git a/dev-requirements.txt b/dev-requirements.txt index 5f393349744..cac7be14f6b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ -git+https://github.com/dbt-labs/dbt-adapters.git@main +git+https://github.com/dbt-labs/dbt-adapters.git@mashumaro_fixes git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git@main +git+https://github.com/dbt-labs/dbt-common.git@mashumaro_fixes git+https://github.com/dbt-labs/dbt-postgres.git@main # black must match what's in .pre-commit-config.yaml to be sure local env matches CI black==24.3.0 diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index 21f2a9a77f5..ffe735f6adf 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -1802,11 +1802,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -2456,11 +2459,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -3250,11 +3256,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -4063,11 +4072,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -4292,11 +4304,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -4306,11 +4321,14 @@ "latest_version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -5430,11 +5448,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -6084,11 +6105,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -7073,11 +7097,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -7916,12 +7943,6 @@ }, "partitions": { "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, { "type": "array", "items": { @@ -7956,6 +7977,12 @@ "additionalProperties": true } }, + { + "type": "array", + "items": { + "type": "string" + } + }, { "type": "null" } @@ -8632,11 +8659,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -9750,11 +9780,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -11599,11 +11632,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -12253,11 +12289,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -13047,11 +13086,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -13860,11 +13902,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -14089,11 +14134,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -14103,11 +14151,14 @@ "latest_version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -15227,11 +15278,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -15881,11 +15935,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -16870,11 +16927,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -17704,12 +17764,6 @@ }, "partitions": { "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, { "type": "array", "items": { @@ -17744,6 +17798,12 @@ "additionalProperties": true } }, + { + "type": "array", + "items": { + "type": "string" + } + }, { "type": "null" } @@ -18218,11 +18278,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -19329,11 +19392,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -19764,11 +19830,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -20491,11 +20560,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -20869,10 +20941,13 @@ "items": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" + }, + { + "type": "string" } ] } @@ -20890,10 +20965,13 @@ "items": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" + }, + { + "type": "string" } ] } @@ -20916,11 +20994,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -21395,11 +21476,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -22129,11 +22213,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } @@ -22514,10 +22601,13 @@ "items": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" + }, + { + "type": "string" } ] } @@ -22535,10 +22625,13 @@ "items": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" + }, + { + "type": "string" } ] } @@ -22561,11 +22654,14 @@ "version": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "number" }, + { + "type": "string" + }, { "type": "null" } diff --git a/tests/functional/artifacts/test_graph_serialization.py b/tests/functional/artifacts/test_graph_serialization.py new file mode 100644 index 00000000000..512ec5ee34c --- /dev/null +++ b/tests/functional/artifacts/test_graph_serialization.py @@ -0,0 +1,48 @@ +import pytest + +from dbt.tests.util import run_dbt + +sources_yml = """ +sources: +- name: TEST + schema: STAGE + tables: + - name: TABLE + external: + partitions: + - name: dl_partition + data_type: string + expression: split_part(METADATA$FILENAME, '/', 2) +""" + +get_partitions_sql = """ +{% macro get_partitions() -%} + {% set source_nodes = graph.sources.values() if graph.sources else [] %} + {% for node in source_nodes %} + {% if node.external %} + {% if node.external.partitions %} + {{print(node.external.partitions)}} + {% endif %} + {% endif %} + {% endfor %} +{%- endmacro %} +""" + + +class TestGraphSerialization: + @pytest.fixture(scope="class") + def models(self): + return { + "sources.yml": sources_yml, + } + + @pytest.fixture(scope="class") + def macros(self): + return {"get_partitions.sql": get_partitions_sql} + + def test_graph_serialization(self, project): + manifest = run_dbt(["parse"]) + assert manifest + assert len(manifest.sources) == 1 + + run_dbt(["run-operation", "get_partitions"]) diff --git a/tests/functional/artifacts/test_serialization.py b/tests/functional/artifacts/test_serialization.py new file mode 100644 index 00000000000..9fa6d8a8fcb --- /dev/null +++ b/tests/functional/artifacts/test_serialization.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Union + +from dbt_common.dataclass_schema import dbtClassMixin + + +@dataclass +class ExternalPartition(dbtClassMixin): + name: str = "" + description: str = "" + data_type: str = "" + meta: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ExternalTable(dbtClassMixin): + location: Optional[str] = None + file_format: Optional[str] = None + row_format: Optional[str] = None + tbl_properties: Optional[str] = None + partitions: Optional[Union[List[ExternalPartition], List[str]]] = None + + +def test_partitions_serialization(): + + part1 = ExternalPartition( + name="partition 1", + description="partition 1", + data_type="string", + ) + + part2 = ExternalPartition( + name="partition 2", + description="partition 2", + data_type="string", + ) + + ext_table = ExternalTable( + location="my_location", + file_format="my file format", + row_format="row format", + partitions=[part1, part2], + ) + + ext_table_dict = ext_table.to_dict() + assert isinstance(ext_table_dict["partitions"][0], dict) + + ext_table_msgpack = ext_table.to_msgpack() + assert ext_table_msgpack diff --git a/tests/functional/exposures/test_exposure_configs.py b/tests/functional/exposures/test_exposure_configs.py index 2ec309623a7..aec50a8523d 100644 --- a/tests/functional/exposures/test_exposure_configs.py +++ b/tests/functional/exposures/test_exposure_configs.py @@ -1,8 +1,8 @@ import pytest from dbt.artifacts.resources import ExposureConfig +from dbt.exceptions import SchemaConfigError from dbt.tests.util import get_manifest, run_dbt, update_config_file -from dbt_common.dataclass_schema import ValidationError from tests.functional.exposures.fixtures import ( disabled_models_exposure_yml, enabled_yaml_level_exposure_yml, @@ -126,7 +126,7 @@ def models(self): } def test_exposure_config_yaml_level(self, project): - with pytest.raises(ValidationError) as excinfo: + with pytest.raises(SchemaConfigError) as excinfo: run_dbt(["parse"]) expected_msg = "'True and False' is not of type 'boolean'" assert expected_msg in str(excinfo.value) diff --git a/tests/functional/metrics/test_metric_configs.py b/tests/functional/metrics/test_metric_configs.py index 2be68d9e17f..a944c1ea5a4 100644 --- a/tests/functional/metrics/test_metric_configs.py +++ b/tests/functional/metrics/test_metric_configs.py @@ -1,9 +1,8 @@ import pytest from dbt.artifacts.resources import MetricConfig -from dbt.exceptions import CompilationError, ParsingError +from dbt.exceptions import CompilationError, ParsingError, SchemaConfigError from dbt.tests.util import get_manifest, run_dbt, update_config_file -from dbt_common.dataclass_schema import ValidationError from tests.functional.metrics.fixtures import ( disabled_metric_level_schema_yml, enabled_metric_level_schema_yml, @@ -170,7 +169,7 @@ def models(self): } def test_invalid_config_metric(self, project): - with pytest.raises(ValidationError) as excinfo: + with pytest.raises(SchemaConfigError) as excinfo: run_dbt(["parse"]) expected_msg = "'True and False' is not of type 'boolean'" assert expected_msg in str(excinfo.value) diff --git a/tests/functional/sources/test_source_configs.py b/tests/functional/sources/test_source_configs.py index 1ceca5d0522..12372c2fbb9 100644 --- a/tests/functional/sources/test_source_configs.py +++ b/tests/functional/sources/test_source_configs.py @@ -1,8 +1,8 @@ import pytest from dbt.artifacts.resources import SourceConfig +from dbt.exceptions import SchemaConfigError from dbt.tests.util import get_manifest, run_dbt, update_config_file -from dbt_common.dataclass_schema import ValidationError from tests.functional.sources.fixtures import ( all_configs_everywhere_schema_yml, all_configs_not_table_schema_yml, @@ -175,7 +175,7 @@ def models(self): } def test_invalid_config_source(self, project): - with pytest.raises(ValidationError) as excinfo: + with pytest.raises(SchemaConfigError) as excinfo: run_dbt(["parse"]) expected_msg = "'True and False' is not of type 'boolean'" assert expected_msg in str(excinfo.value) diff --git a/tests/unit/context/test_context.py b/tests/unit/context/test_context.py index 10e591093ee..2580f7b692d 100644 --- a/tests/unit/context/test_context.py +++ b/tests/unit/context/test_context.py @@ -473,7 +473,7 @@ def test_model_parse_context(config_postgres, manifest_fx, get_adapter, get_incl model=mock_model(), config=config_postgres, manifest=manifest_fx, - context_config=mock.MagicMock(), + config_builder=mock.MagicMock(), ) assert_has_keys(REQUIRED_MODEL_KEYS, MAYBE_KEYS, ctx) diff --git a/tests/unit/parser/test_parser.py b/tests/unit/parser/test_parser.py index 8894e47ce84..88b8f9414fe 100644 --- a/tests/unit/parser/test_parser.py +++ b/tests/unit/parser/test_parser.py @@ -8,7 +8,7 @@ from dbt import tracking from dbt.artifacts.resources import ModelConfig, RefArgs -from dbt.context.context_config import ContextConfig +from dbt.context.context_config import ConfigBuilder from dbt.contracts.files import FileHash, FilePath, SchemaSourceFile, SourceFile from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.model_config import NodeConfig, SnapshotConfig, TestConfig @@ -1285,7 +1285,7 @@ def setUp(self): checksum=None, unrendered_config={"materialized": "table"}, ) - self.example_config = ContextConfig( + self.example_config_builder = ConfigBuilder( self.root_project_config, self.example_node.fqn, self.example_node.resource_type, @@ -1315,90 +1315,92 @@ def test_source_shifting(self): def test_sample_results(self): # --- missed ref --- # node = deepcopy(self.example_node) - config = deepcopy(self.example_config) + config_builder = deepcopy(self.example_config_builder) sample_node = deepcopy(self.example_node) - sample_config = deepcopy(self.example_config) + sample_config_builder = deepcopy(self.example_config_builder) sample_node.refs = [] node.refs = ["myref"] - result = _get_sample_result(sample_node, sample_config, node, config) + result = _get_sample_result(sample_node, sample_config_builder, node, config_builder) self.assertEqual([(7, "missed_ref_value")], result) # --- false positive ref --- # node = deepcopy(self.example_node) - config = deepcopy(self.example_config) + config_builder = deepcopy(self.example_config_builder) sample_node = deepcopy(self.example_node) - sample_config = deepcopy(self.example_config) + sample_config_builder = deepcopy(self.example_config_builder) sample_node.refs = ["myref"] node.refs = [] - result = _get_sample_result(sample_node, sample_config, node, config) + result = _get_sample_result(sample_node, sample_config_builder, node, config_builder) self.assertEqual([(6, "false_positive_ref_value")], result) # --- missed source --- # node = deepcopy(self.example_node) - config = deepcopy(self.example_config) + config_builder = deepcopy(self.example_config_builder) sample_node = deepcopy(self.example_node) - sample_config = deepcopy(self.example_config) + sample_config_builder = deepcopy(self.example_config_builder) sample_node.sources = [] node.sources = [["abc", "def"]] - result = _get_sample_result(sample_node, sample_config, node, config) + result = _get_sample_result(sample_node, sample_config_builder, node, config_builder) self.assertEqual([(5, "missed_source_value")], result) # --- false positive source --- # node = deepcopy(self.example_node) - config = deepcopy(self.example_config) + config_builder = deepcopy(self.example_config_builder) sample_node = deepcopy(self.example_node) - sample_config = deepcopy(self.example_config) + sample_config_builder = deepcopy(self.example_config_builder) sample_node.sources = [["abc", "def"]] node.sources = [] - result = _get_sample_result(sample_node, sample_config, node, config) + result = _get_sample_result(sample_node, sample_config_builder, node, config_builder) self.assertEqual([(4, "false_positive_source_value")], result) # --- missed config --- # node = deepcopy(self.example_node) - config = deepcopy(self.example_config) + config_builder = deepcopy(self.example_config_builder) sample_node = deepcopy(self.example_node) - sample_config = deepcopy(self.example_config) + sample_config_builder = deepcopy(self.example_config_builder) - sample_config._config_call_dict = {} - config._config_call_dict = {"key": "value"} + sample_config_builder._config_call_dict = {} + config_builder._config_call_dict = {"key": "value"} - result = _get_sample_result(sample_node, sample_config, node, config) + result = _get_sample_result(sample_node, sample_config_builder, node, config_builder) self.assertEqual([(3, "missed_config_value")], result) # --- false positive config --- # node = deepcopy(self.example_node) - config = deepcopy(self.example_config) + config_builder = deepcopy(self.example_config_builder) sample_node = deepcopy(self.example_node) - sample_config = deepcopy(self.example_config) + sample_config_builder = deepcopy(self.example_config_builder) - sample_config._config_call_dict = {"key": "value"} - config._config_call_dict = {} + sample_config_builder._config_call_dict = {"key": "value"} + config_builder._config_call_dict = {} - result = _get_sample_result(sample_node, sample_config, node, config) + result = _get_sample_result(sample_node, sample_config_builder, node, config_builder) self.assertEqual([(2, "false_positive_config_value")], result) def test_exp_sample_results(self): node = deepcopy(self.example_node) - config = deepcopy(self.example_config) + config_builder = deepcopy(self.example_config_builder) sample_node = deepcopy(self.example_node) - sample_config = deepcopy(self.example_config) - result = _get_exp_sample_result(sample_node, sample_config, node, config) + sample_config_builder = deepcopy(self.example_config_builder) + result = _get_exp_sample_result(sample_node, sample_config_builder, node, config_builder) self.assertEqual(["00_experimental_exact_match"], result) def test_stable_sample_results(self): node = deepcopy(self.example_node) - config = deepcopy(self.example_config) + config_builder = deepcopy(self.example_config_builder) sample_node = deepcopy(self.example_node) - sample_config = deepcopy(self.example_config) - result = _get_stable_sample_result(sample_node, sample_config, node, config) + sample_config_builder = deepcopy(self.example_config_builder) + result = _get_stable_sample_result( + sample_node, sample_config_builder, node, config_builder + ) self.assertEqual(["80_stable_exact_match"], result) diff --git a/tests/unit/utils/manifest.py b/tests/unit/utils/manifest.py index 0950f68ebb5..26b564209be 100644 --- a/tests/unit/utils/manifest.py +++ b/tests/unit/utils/manifest.py @@ -470,7 +470,7 @@ def make_semantic_model( return SemanticModel( name=name, resource_type=NodeType.SemanticModel, - model=model, + model=model.name, node_relation=NodeRelation( alias=model.alias, schema_name="dbt",