diff --git a/.changes/unreleased/Features-20240729-173203.yaml b/.changes/unreleased/Features-20240729-173203.yaml new file mode 100644 index 00000000000..e788c8e9cc9 --- /dev/null +++ b/.changes/unreleased/Features-20240729-173203.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Include models that depend on changed vars in state:modified, add state:modified.vars + selection method +time: 2024-07-29T17:32:03.368508-04:00 +custom: + Author: michelleark + Issue: "4304" diff --git a/.changes/unreleased/Fixes-20240923-190758.yaml b/.changes/unreleased/Fixes-20240923-190758.yaml new file mode 100644 index 00000000000..4d005ec5999 --- /dev/null +++ b/.changes/unreleased/Fixes-20240923-190758.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Allow singular tests to be documented in properties.yml +time: 2024-09-23T19:07:58.151069+01:00 +custom: + Author: aranke + Issue: "9005" diff --git a/core/dbt/artifacts/resources/v1/components.py b/core/dbt/artifacts/resources/v1/components.py index 8eb43f35d8e..02bfa5d875d 100644 --- a/core/dbt/artifacts/resources/v1/components.py +++ b/core/dbt/artifacts/resources/v1/components.py @@ -197,6 +197,7 @@ class ParsedResource(ParsedResourceMandatory): unrendered_config_call_dict: Dict[str, Any] = field(default_factory=dict) relation_name: Optional[str] = None raw_code: str = "" + vars: Dict[str, Any] = field(default_factory=dict) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) diff --git a/core/dbt/artifacts/resources/v1/exposure.py b/core/dbt/artifacts/resources/v1/exposure.py index 00f3c8b89e1..7d8c291381b 100644 --- a/core/dbt/artifacts/resources/v1/exposure.py +++ b/core/dbt/artifacts/resources/v1/exposure.py @@ -41,6 +41,7 @@ class Exposure(GraphResource): tags: List[str] = field(default_factory=list) config: ExposureConfig = field(default_factory=ExposureConfig) unrendered_config: Dict[str, Any] = field(default_factory=dict) + vars: Dict[str, Any] = field(default_factory=dict) url: Optional[str] = None depends_on: DependsOn = field(default_factory=DependsOn) refs: List[RefArgs] = field(default_factory=list) diff --git a/core/dbt/artifacts/resources/v1/source_definition.py b/core/dbt/artifacts/resources/v1/source_definition.py index 6c1c3679a00..2650486baa0 100644 --- a/core/dbt/artifacts/resources/v1/source_definition.py +++ b/core/dbt/artifacts/resources/v1/source_definition.py @@ -69,5 +69,6 @@ class SourceDefinition(ParsedSourceMandatory): config: SourceConfig = field(default_factory=SourceConfig) patch_path: Optional[str] = None unrendered_config: Dict[str, Any] = field(default_factory=dict) + vars: Dict[str, Any] = field(default_factory=dict) relation_name: Optional[str] = None created_at: float = field(default_factory=lambda: time.time()) diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 25b0f343ef2..cbad5a38434 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -158,14 +158,8 @@ def _parse_versions(versions: Union[List[str], str]) -> List[VersionSpecifier]: return [VersionSpecifier.from_version_string(v) for v in versions] -def _all_source_paths( - model_paths: List[str], - seed_paths: List[str], - snapshot_paths: List[str], - analysis_paths: List[str], - macro_paths: List[str], -) -> List[str]: - paths = chain(model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths) +def _all_source_paths(*args: List[str]) -> List[str]: + paths = chain(*args) # Strip trailing slashes since the path is the same even though the name is not stripped_paths = map(lambda s: s.rstrip("/"), paths) return list(set(stripped_paths)) @@ -409,7 +403,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": snapshot_paths: List[str] = value_or(cfg.snapshot_paths, ["snapshots"]) all_source_paths: List[str] = _all_source_paths( - model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths + model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths, test_paths ) docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths) @@ -652,6 +646,7 @@ def all_source_paths(self) -> List[str]: self.snapshot_paths, self.analysis_paths, self.macro_paths, + self.test_paths, ) @property diff --git a/core/dbt/context/configured.py b/core/dbt/context/configured.py index 240d9afb843..9ee7ebc79c3 100644 --- a/core/dbt/context/configured.py +++ b/core/dbt/context/configured.py @@ -31,23 +31,35 @@ def __init__(self, package_name: str): self.resource_type = NodeType.Model +class SchemaYamlVars: + def __init__(self): + self.env_vars = {} + self.vars = {} + + class ConfiguredVar(Var): def __init__( self, context: Dict[str, Any], config: AdapterRequiredConfig, project_name: str, + schema_yaml_vars: Optional[SchemaYamlVars] = None, ): super().__init__(context, config.cli_vars) self._config = config self._project_name = project_name + self.schema_yaml_vars = schema_yaml_vars def __call__(self, var_name, default=Var._VAR_NOTSET): my_config = self._config.load_dependencies()[self._project_name] + var_found = False + var_value = None + # cli vars > active project > local project if var_name in self._config.cli_vars: - return self._config.cli_vars[var_name] + var_found = True + var_value = self._config.cli_vars[var_name] adapter_type = self._config.credentials.type lookup = FQNLookup(self._project_name) @@ -58,19 +70,21 @@ def __call__(self, var_name, default=Var._VAR_NOTSET): all_vars.add(my_config.vars.vars_for(lookup, adapter_type)) all_vars.add(active_vars) - if var_name in all_vars: - return all_vars[var_name] + if not var_found and var_name in all_vars: + var_found = True + var_value = all_vars[var_name] - if default is not Var._VAR_NOTSET: - return default - - return self.get_missing_var(var_name) + if not var_found and default is not Var._VAR_NOTSET: + var_found = True + var_value = default + if not var_found: + return self.get_missing_var(var_name) + else: + if self.schema_yaml_vars: + self.schema_yaml_vars.vars[var_name] = var_value -class SchemaYamlVars: - def __init__(self): - self.env_vars = {} - self.vars = {} + return var_value class SchemaYamlContext(ConfiguredContext): @@ -82,7 +96,7 @@ def __init__(self, config, project_name: str, schema_yaml_vars: Optional[SchemaY @contextproperty() def var(self) -> ConfiguredVar: - return ConfiguredVar(self._ctx, self.config, self._project_name) + return ConfiguredVar(self._ctx, self.config, self._project_name, self.schema_yaml_vars) @contextmember() def env_var(self, var: str, default: Optional[str] = None) -> str: diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index dfc8c9bb40b..a0e3751587a 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -790,6 +790,14 @@ def get_missing_var(self, var_name): # in the parser, just always return None. return None + def __call__(self, var_name: str, default: Any = ModelConfiguredVar._VAR_NOTSET) -> Any: + var_value = super().__call__(var_name, default) + + if self._node and hasattr(self._node, "vars"): + self._node.vars[var_name] = var_value + + return var_value + class RuntimeVar(ModelConfiguredVar): pass diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index 04d7909133a..e6a17adad24 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -212,6 +212,7 @@ class SchemaSourceFile(BaseSourceFile): # created too, but those are in 'sources' sop: List[SourceKey] = field(default_factory=list) env_vars: Dict[str, Any] = field(default_factory=dict) + vars: Dict[str, Any] = field(default_factory=dict) unrendered_configs: Dict[str, Any] = field(default_factory=dict) pp_dict: Optional[Dict[str, Any]] = None pp_test_index: Optional[Dict[str, Any]] = None @@ -318,6 +319,22 @@ def get_all_test_ids(self): test_ids.extend(self.data_tests[key][name]) return test_ids + def add_vars(self, vars: Dict[str, Any], yaml_key: str, name: str) -> None: + if yaml_key not in self.vars: + self.vars[yaml_key] = {} + + if name not in self.vars[yaml_key]: + self.vars[yaml_key][name] = vars + + def get_vars(self, yaml_key: str, name: str) -> Dict[str, Any]: + if yaml_key not in self.vars: + return {} + + if name not in self.vars[yaml_key]: + return {} + + return self.vars[yaml_key][name] + def add_unrendered_config(self, unrendered_config, yaml_key, name, version=None): versioned_name = f"{name}_v{version}" if version is not None else name diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index f4cdafea737..b556b479fb4 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -58,6 +58,7 @@ SavedQuery, SeedNode, SemanticModel, + SingularTestNode, SourceDefinition, UnitTestDefinition, UnitTestFileFixture, @@ -89,7 +90,7 @@ RefName = str -def find_unique_id_for_package(storage, key, package: Optional[PackageName]): +def find_unique_id_for_package(storage, key, package: Optional[PackageName]) -> Optional[UniqueID]: if key not in storage: return None @@ -470,6 +471,43 @@ class AnalysisLookup(RefableLookup): _versioned_types: ClassVar[set] = set() +class SingularTestLookup(dbtClassMixin): + def __init__(self, manifest: "Manifest") -> None: + self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} + self.populate(manifest) + + def get_unique_id(self, search_name, package: Optional[PackageName]) -> Optional[UniqueID]: + return find_unique_id_for_package(self.storage, search_name, package) + + def find( + self, search_name, package: Optional[PackageName], manifest: "Manifest" + ) -> Optional[SingularTestNode]: + unique_id = self.get_unique_id(search_name, package) + if unique_id is not None: + return self.perform_lookup(unique_id, manifest) + return None + + def add_singular_test(self, source: SingularTestNode) -> None: + if source.search_name not in self.storage: + self.storage[source.search_name] = {} + + self.storage[source.search_name][source.package_name] = source.unique_id + + def populate(self, manifest: "Manifest") -> None: + for node in manifest.nodes.values(): + if isinstance(node, SingularTestNode): + self.add_singular_test(node) + + def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SingularTestNode: + if unique_id not in manifest.nodes: + raise dbt_common.exceptions.DbtInternalError( + f"Singular test {unique_id} found in cache but not found in manifest" + ) + node = manifest.nodes[unique_id] + assert isinstance(node, SingularTestNode) + return node + + def _packages_to_search( current_project: str, node_package: str, @@ -869,6 +907,9 @@ class Manifest(MacroMethods, dbtClassMixin): _analysis_lookup: Optional[AnalysisLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) + _singular_test_lookup: Optional[SingularTestLookup] = field( + default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} + ) _parsing_info: ParsingInfo = field( default_factory=ParsingInfo, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}, @@ -1264,6 +1305,12 @@ def analysis_lookup(self) -> AnalysisLookup: self._analysis_lookup = AnalysisLookup(self) return self._analysis_lookup + @property + def singular_test_lookup(self) -> SingularTestLookup: + if self._singular_test_lookup is None: + self._singular_test_lookup = SingularTestLookup(self) + return self._singular_test_lookup + @property def external_node_unique_ids(self): return [node.unique_id for node in self.nodes.values() if node.is_external_node] @@ -1708,6 +1755,7 @@ def __reduce_ex__(self, protocol): self._semantic_model_by_measure_lookup, self._disabled_lookup, self._analysis_lookup, + self._singular_test_lookup, ) return self.__class__, args diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index f4b907a9a3c..1dd39da1c9d 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -369,6 +369,12 @@ def same_contract(self, old, adapter_type=None) -> bool: # This would only apply to seeds return True + def same_vars(self, old) -> bool: + if get_flags().state_modified_compare_vars: + return self.vars == old.vars + else: + return True + def same_contents(self, old, adapter_type) -> bool: if old is None: return False @@ -382,6 +388,7 @@ def same_contents(self, old, adapter_type) -> bool: and self.same_persisted_description(old) and self.same_fqn(old) and self.same_database_representation(old) + and self.same_vars(old) and same_contract and True ) @@ -1251,6 +1258,12 @@ def same_config(self, old: "SourceDefinition") -> bool: old.unrendered_config, ) + def same_vars(self, other: "SourceDefinition") -> bool: + if get_flags().state_modified_compare_vars: + return self.vars == other.vars + else: + return True + def same_contents(self, old: Optional["SourceDefinition"]) -> bool: # existing when it didn't before is a change! if old is None: @@ -1271,6 +1284,7 @@ def same_contents(self, old: Optional["SourceDefinition"]) -> bool: and self.same_quoting(old) and self.same_freshness(old) and self.same_external(old) + and self.same_vars(old) and True ) @@ -1367,6 +1381,12 @@ def same_config(self, old: "Exposure") -> bool: old.unrendered_config, ) + def same_vars(self, old: "Exposure") -> bool: + if get_flags().state_modified_compare_vars: + return self.vars == old.vars + else: + return True + def same_contents(self, old: Optional["Exposure"]) -> bool: # existing when it didn't before is a change! # metadata/tags changes are not "changes" @@ -1383,6 +1403,7 @@ def same_contents(self, old: Optional["Exposure"]) -> bool: and self.same_label(old) and self.same_depends_on(old) and self.same_config(old) + and self.same_vars(old) and True ) @@ -1634,6 +1655,7 @@ class ParsedNodePatch(ParsedPatch): latest_version: Optional[NodeVersion] constraints: List[Dict[str, Any]] deprecation_date: Optional[datetime] + vars: Dict[str, Any] time_spine: Optional[TimeSpine] = None @@ -1642,6 +1664,11 @@ class ParsedMacroPatch(ParsedPatch): arguments: List[MacroArgument] = field(default_factory=list) +@dataclass +class ParsedSingularTestPatch(ParsedPatch): + pass + + # ==================================== # Node unions/categories # ==================================== diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 847be3d3a2a..ebe704fc1c5 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -202,6 +202,11 @@ class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasColumnProps, HasYamlMe access: Optional[str] = None +@dataclass +class UnparsedSingularTestUpdate(HasConfig, HasColumnProps, HasYamlMetadata): + pass + + @dataclass class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasColumnAndTestProps, HasYamlMetadata): quote_columns: Optional[bool] = None diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index e08131ecd8f..198b6c6325b 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -342,6 +342,7 @@ class ProjectFlags(ExtensibleDbtClassMixin): require_resource_names_without_spaces: bool = False source_freshness_run_project_hooks: bool = False state_modified_compare_more_unrendered_values: bool = False + state_modified_compare_vars: bool = False @property def project_only_flags(self) -> Dict[str, Any]: @@ -350,6 +351,7 @@ def project_only_flags(self) -> Dict[str, Any]: "require_resource_names_without_spaces": self.require_resource_names_without_spaces, "source_freshness_run_project_hooks": self.source_freshness_run_project_hooks, "state_modified_compare_more_unrendered_values": self.state_modified_compare_more_unrendered_values, + "state_modified_compare_vars": self.state_modified_compare_vars, } diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index dbeaf7ed4c3..c7a2490760b 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -752,6 +752,7 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu "modified.relation": self.check_modified_factory("same_database_representation"), "modified.macros": self.check_modified_macros, "modified.contract": self.check_modified_contract("same_contract", adapter_type), + "modified.vars": self.check_modified_factory("same_vars"), } if selector in state_checks: checker = state_checks[selector] diff --git a/core/dbt/parser/common.py b/core/dbt/parser/common.py index 66d84d2db9b..5cc4385ea1c 100644 --- a/core/dbt/parser/common.py +++ b/core/dbt/parser/common.py @@ -13,6 +13,7 @@ UnparsedMacroUpdate, UnparsedModelUpdate, UnparsedNodeUpdate, + UnparsedSingularTestUpdate, ) from dbt.exceptions import ParsingError from dbt.node_types import NodeType @@ -58,6 +59,7 @@ def trimmed(inp: str) -> str: UnpatchedSourceDefinition, UnparsedExposure, UnparsedModelUpdate, + UnparsedSingularTestUpdate, ) diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index dc99e87a218..4fe28a74866 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -91,6 +91,9 @@ def parse_exposure(self, unparsed: UnparsedExposure) -> None: unique_id = f"{NodeType.Exposure}.{package_name}.{unparsed.name}" path = self.yaml.path.relative_path + assert isinstance(self.yaml.file, SchemaSourceFile) + exposure_vars = self.yaml.file.get_vars(self.key, unparsed.name) + fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) @@ -133,6 +136,7 @@ def parse_exposure(self, unparsed: UnparsedExposure) -> None: maturity=unparsed.maturity, config=config, unrendered_config=unrendered_config, + vars=exposure_vars, ) ctx = generate_parse_exposure( parsed, @@ -144,7 +148,6 @@ def parse_exposure(self, unparsed: UnparsedExposure) -> None: get_rendered(depends_on_jinja, ctx, parsed, capture_macros=True) # parsed now has a populated refs/sources/metrics - assert isinstance(self.yaml.file, SchemaSourceFile) if parsed.config.enabled: self.manifest.add_exposure(self.yaml.file, parsed) else: diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 29f36f96b34..eb1a66e6d12 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -17,6 +17,7 @@ ModelNode, ParsedMacroPatch, ParsedNodePatch, + ParsedSingularTestPatch, UnpatchedSourceDefinition, ) from dbt.contracts.graph.unparsed import ( @@ -27,6 +28,7 @@ UnparsedMacroUpdate, UnparsedModelUpdate, UnparsedNodeUpdate, + UnparsedSingularTestUpdate, UnparsedSourceDefinition, ) from dbt.events.types import ( @@ -207,6 +209,10 @@ def parse_file(self, block: FileBlock, dct: Optional[Dict] = None) -> None: parser = MacroPatchParser(self, yaml_block, "macros") parser.parse() + if "data_tests" in dct: + parser = SingularTestPatchParser(self, yaml_block, "data_tests") + parser.parse() + # PatchParser.parse() (but never test_blocks) if "analyses" in dct: parser = AnalysisPatchParser(self, yaml_block, "analyses") @@ -301,7 +307,9 @@ def _add_yaml_snapshot_nodes_to_manifest( self.manifest.rebuild_ref_lookup() -Parsed = TypeVar("Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch) +Parsed = TypeVar( + "Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch, ParsedSingularTestPatch +) NodeTarget = TypeVar("NodeTarget", UnparsedNodeUpdate, UnparsedAnalysisUpdate, UnparsedModelUpdate) NonSourceTarget = TypeVar( "NonSourceTarget", @@ -309,6 +317,7 @@ def _add_yaml_snapshot_nodes_to_manifest( UnparsedAnalysisUpdate, UnparsedMacroUpdate, UnparsedModelUpdate, + UnparsedSingularTestUpdate, ) @@ -403,10 +412,14 @@ def get_key_dicts(self) -> Iterable[Dict[str, Any]]: if self.schema_yaml_vars.env_vars: self.schema_parser.manifest.env_vars.update(self.schema_yaml_vars.env_vars) - for var in self.schema_yaml_vars.env_vars.keys(): - schema_file.add_env_var(var, self.key, entry["name"]) + for env_var in self.schema_yaml_vars.env_vars.keys(): + schema_file.add_env_var(env_var, self.key, entry["name"]) self.schema_yaml_vars.env_vars = {} + if self.schema_yaml_vars.vars: + schema_file.add_vars(self.schema_yaml_vars.vars, self.key, entry["name"]) + self.schema_yaml_vars.vars = {} + yield entry def render_entry(self, dct): @@ -677,6 +690,9 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: # code consistency. deprecation_date: Optional[datetime.datetime] = None time_spine: Optional[TimeSpine] = None + assert isinstance(self.yaml.file, SchemaSourceFile) + source_file: SchemaSourceFile = self.yaml.file + if isinstance(block.target, UnparsedModelUpdate): deprecation_date = block.target.deprecation_date time_spine = ( @@ -709,9 +725,9 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: constraints=block.target.constraints, deprecation_date=deprecation_date, time_spine=time_spine, + vars=source_file.get_vars(block.target.yaml_key, block.target.name), ) - assert isinstance(self.yaml.file, SchemaSourceFile) - source_file: SchemaSourceFile = self.yaml.file + if patch.yaml_key in ["models", "seeds", "snapshots"]: unique_id = self.manifest.ref_lookup.get_unique_id( patch.name, self.project.project_name, None @@ -805,6 +821,8 @@ def patch_node_properties(self, node, patch: "ParsedNodePatch") -> None: node.description = patch.description node.columns = patch.columns node.name = patch.name + # Prefer node-level vars to vars from patch + node.vars = {**patch.vars, **node.vars} if not isinstance(node, ModelNode): for attr in ["latest_version", "access", "version", "constraints"]: @@ -954,6 +972,7 @@ def parse_patch(self, block: TargetBlock[UnparsedModelUpdate], refs: ParserRef) latest_version=latest_version, constraints=unparsed_version.constraints or target.constraints, deprecation_date=unparsed_version.deprecation_date, + vars=source_file.get_vars(block.target.yaml_key, block.target.name), ) # Node patched before config because config patching depends on model name, # which may have been updated in the version patch @@ -1116,6 +1135,55 @@ def _target_type(self) -> Type[UnparsedAnalysisUpdate]: return UnparsedAnalysisUpdate +class SingularTestPatchParser(PatchParser[UnparsedSingularTestUpdate, ParsedSingularTestPatch]): + def get_block(self, node: UnparsedSingularTestUpdate) -> TargetBlock: + return TargetBlock.from_yaml_block(self.yaml, node) + + def _target_type(self) -> Type[UnparsedSingularTestUpdate]: + return UnparsedSingularTestUpdate + + def parse_patch(self, block: TargetBlock[UnparsedSingularTestUpdate], refs: ParserRef) -> None: + patch = ParsedSingularTestPatch( + name=block.target.name, + description=block.target.description, + meta=block.target.meta, + docs=block.target.docs, + config=block.target.config, + original_file_path=block.target.original_file_path, + yaml_key=block.target.yaml_key, + package_name=block.target.package_name, + ) + + assert isinstance(self.yaml.file, SchemaSourceFile) + source_file: SchemaSourceFile = self.yaml.file + + unique_id = self.manifest.singular_test_lookup.get_unique_id( + block.name, block.target.package_name + ) + if not unique_id: + warn_or_error( + NoNodeForYamlKey( + patch_name=patch.name, + yaml_key=patch.yaml_key, + file_path=source_file.path.original_file_path, + ) + ) + return + + node = self.manifest.nodes.get(unique_id) + assert node is not None + + source_file.append_patch(patch.yaml_key, unique_id) + if patch.config: + self.patch_node_config(node, patch) + + node.patch_path = patch.file_id + node.description = patch.description + node.created_at = time.time() + node.meta = patch.meta + node.docs = patch.docs + + class MacroPatchParser(PatchParser[UnparsedMacroUpdate, ParsedMacroPatch]): def get_block(self, node: UnparsedMacroUpdate) -> TargetBlock: return TargetBlock.from_yaml_block(self.yaml, node) diff --git a/core/dbt/parser/sources.py b/core/dbt/parser/sources.py index 68dbed94ce5..4aa08787b37 100644 --- a/core/dbt/parser/sources.py +++ b/core/dbt/parser/sources.py @@ -12,6 +12,7 @@ ContextConfigGenerator, UnrenderedConfigGenerator, ) +from dbt.contracts.files import SchemaSourceFile from dbt.contracts.graph.manifest import Manifest, SourceKey from dbt.contracts.graph.nodes import ( GenericTestNode, @@ -158,6 +159,10 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> SourceDefinition: rendered=False, ) + schema_file = self.manifest.files[target.file_id] + assert isinstance(schema_file, SchemaSourceFile) + source_vars = schema_file.get_vars("sources", source.name) + if not isinstance(config, SourceConfig): raise DbtInternalError( f"Calculated a {type(config)} for a source, but expected a SourceConfig" @@ -190,6 +195,7 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> SourceDefinition: tags=tags, config=config, unrendered_config=unrendered_config, + vars=source_vars, ) if ( diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index d1246d526ce..ea21214b4fc 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -721,6 +721,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "root_path": { "anyOf": [ { @@ -1754,6 +1760,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -2402,6 +2414,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -3187,6 +3205,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -3991,6 +4015,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -5346,6 +5376,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -5994,6 +6030,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -6946,6 +6988,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -8083,6 +8131,12 @@ "type": "string" } }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "relation_name": { "anyOf": [ { @@ -8454,6 +8508,12 @@ "type": "string" } }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "url": { "anyOf": [ { @@ -10525,6 +10585,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "root_path": { "anyOf": [ { @@ -11558,6 +11624,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -12206,6 +12278,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -12991,6 +13069,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -13795,6 +13879,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -15150,6 +15240,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -15798,6 +15894,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -16750,6 +16852,12 @@ "type": "string", "default": "" }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -17878,6 +17986,12 @@ "type": "string" } }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "relation_name": { "anyOf": [ { @@ -18047,6 +18161,12 @@ "type": "string" } }, + "vars": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "url": { "anyOf": [ { diff --git a/tests/functional/artifacts/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py index 606a3e8f5d7..368d6fec87d 100644 --- a/tests/functional/artifacts/expected_manifest.py +++ b/tests/functional/artifacts/expected_manifest.py @@ -363,6 +363,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "extra_ctes": [], "checksum": checksum_file(model_sql_path), "unrendered_config": unrendered_model_config, + "vars": {}, "access": "protected", "version": None, "latest_version": None, @@ -462,6 +463,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "extra_ctes": [], "checksum": checksum_file(second_model_sql_path), "unrendered_config": unrendered_second_config, + "vars": {}, "access": "protected", "version": None, "latest_version": None, @@ -544,6 +546,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "docs": {"node_color": None, "show": True}, "checksum": checksum_file(seed_path), "unrendered_config": unrendered_seed_config, + "vars": {}, "relation_name": relation_name_node_format.format( project.database, my_schema_name, "seed" ), @@ -599,6 +602,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "vars": {}, "contract": {"checksum": None, "enforced": False, "alias_types": True}, }, "snapshot.test.snapshot_seed": { @@ -646,6 +650,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "unique_id": "snapshot.test.snapshot_seed", "unrendered_config": unrendered_snapshot_config, + "vars": {"alternate_schema": alternate_schema}, }, "test.test.test_nothing_model_.5d38568946": { "alias": "test_nothing_model_", @@ -698,6 +703,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "vars": {}, }, "test.test.unique_model_id.67b76558ff": { "alias": "unique_model_id", @@ -751,6 +757,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "vars": {}, }, }, "sources": { @@ -807,6 +814,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "unrendered_config": {}, + "vars": {}, }, }, "exposures": { @@ -841,6 +849,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "unique_id": "exposure.test.notebook_exposure", "url": "http://example.com/notebook/1", "unrendered_config": {}, + "vars": {}, }, "exposure.test.simple_exposure": { "created_at": ANY, @@ -873,6 +882,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "tags": [], "unrendered_config": {}, + "vars": {}, }, }, "metrics": {}, @@ -990,6 +1000,7 @@ def expected_references_manifest(project): "extra_ctes": [], "checksum": checksum_file(ephemeral_copy_path), "unrendered_config": get_unrendered_model_config(materialized="ephemeral"), + "vars": {}, "access": "protected", "version": None, "latest_version": None, @@ -1062,6 +1073,7 @@ def expected_references_manifest(project): "unrendered_config": get_unrendered_model_config( materialized="table", group="test_group" ), + "vars": {}, "access": "protected", "version": None, "latest_version": None, @@ -1130,6 +1142,7 @@ def expected_references_manifest(project): "extra_ctes": [], "checksum": checksum_file(view_summary_path), "unrendered_config": get_unrendered_model_config(materialized="view"), + "vars": {}, "access": "protected", "version": None, "latest_version": None, @@ -1213,6 +1226,7 @@ def expected_references_manifest(project): "unique_id": "seed.test.seed", "checksum": checksum_file(seed_path), "unrendered_config": get_unrendered_seed_config(), + "vars": {}, "relation_name": '"{0}"."{1}".seed'.format(project.database, my_schema_name), }, "snapshot.test.snapshot_seed": { @@ -1227,7 +1241,10 @@ def expected_references_manifest(project): "config": get_rendered_snapshot_config(target_schema=alternate_schema), "contract": {"checksum": None, "enforced": False, "alias_types": True}, "database": model_database, - "depends_on": {"macros": [], "nodes": ["seed.test.seed"]}, + "depends_on": { + "macros": [], + "nodes": ["seed.test.seed"], + }, "description": "", "docs": {"node_color": None, "show": True}, "extra_ctes": [], @@ -1255,6 +1272,7 @@ def expected_references_manifest(project): "unrendered_config": get_unrendered_snapshot_config( target_schema=alternate_schema ), + "vars": {"alternate_schema": alternate_schema}, }, }, "sources": { @@ -1309,6 +1327,7 @@ def expected_references_manifest(project): "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "unrendered_config": {}, + "vars": {}, }, }, "exposures": { @@ -1334,6 +1353,7 @@ def expected_references_manifest(project): "package_name": "test", "path": "schema.yml", "refs": [{"name": "view_summary", "package": None, "version": None}], + "vars": {}, "resource_type": "exposure", "sources": [], "type": "notebook", @@ -1594,6 +1614,7 @@ def expected_versions_manifest(project): group="test_group", meta={"size": "large", "color": "blue"}, ), + "vars": {}, "access": "protected", "version": 1, "latest_version": 2, @@ -1665,6 +1686,7 @@ def expected_versions_manifest(project): "unrendered_config": get_unrendered_model_config( materialized="view", group="test_group", meta={"size": "large", "color": "red"} ), + "vars": {}, "access": "protected", "version": 2, "latest_version": 2, @@ -1723,6 +1745,7 @@ def expected_versions_manifest(project): "extra_ctes": [], "checksum": checksum_file(ref_versioned_model_path), "unrendered_config": get_unrendered_model_config(), + "vars": {}, "access": "protected", "version": None, "latest_version": None, @@ -1780,6 +1803,7 @@ def expected_versions_manifest(project): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "vars": {}, }, "test.test.unique_versioned_model_v1_count.0b4c0b688a": { "alias": "unique_versioned_model_v1_count", @@ -1833,6 +1857,7 @@ def expected_versions_manifest(project): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "vars": {}, }, "test.test.unique_versioned_model_v2_first_name.998430d28e": { "alias": "unique_versioned_model_v2_first_name", @@ -1886,6 +1911,7 @@ def expected_versions_manifest(project): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "vars": {}, }, }, "exposures": { @@ -1917,6 +1943,7 @@ def expected_versions_manifest(project): "unique_id": "exposure.test.notebook_exposure", "url": None, "unrendered_config": {}, + "vars": {}, }, }, "metrics": {}, diff --git a/tests/functional/data_test_patch/fixtures.py b/tests/functional/data_test_patch/fixtures.py new file mode 100644 index 00000000000..53d297d34bd --- /dev/null +++ b/tests/functional/data_test_patch/fixtures.py @@ -0,0 +1,32 @@ +tests__my_singular_test_sql = """ +with my_cte as ( + select 1 as id, 'foo' as name + union all + select 2 as id, 'bar' as name +) +select * from my_cte +""" + +tests__schema_yml = """ +data_tests: + - name: my_singular_test + description: "{{ doc('my_singular_test_documentation') }}" + config: + error_if: ">10" + meta: + some_key: some_val +""" + +tests__doc_block_md = """ +{% docs my_singular_test_documentation %} + +Some docs from a doc block + +{% enddocs %} +""" + +tests__invalid_name_schema_yml = """ +data_tests: + - name: my_double_test + description: documentation, but make it double +""" diff --git a/tests/functional/data_test_patch/test_singular_test_patch.py b/tests/functional/data_test_patch/test_singular_test_patch.py new file mode 100644 index 00000000000..a57f93b50c3 --- /dev/null +++ b/tests/functional/data_test_patch/test_singular_test_patch.py @@ -0,0 +1,53 @@ +import os + +import pytest + +from dbt.tests.util import get_artifact, run_dbt, run_dbt_and_capture +from tests.functional.data_test_patch.fixtures import ( + tests__doc_block_md, + tests__invalid_name_schema_yml, + tests__my_singular_test_sql, + tests__schema_yml, +) + + +class TestPatchSingularTest: + @pytest.fixture(scope="class") + def tests(self): + return { + "my_singular_test.sql": tests__my_singular_test_sql, + "schema.yml": tests__schema_yml, + "doc_block.md": tests__doc_block_md, + } + + def test_compile(self, project): + run_dbt(["compile"]) + manifest = get_artifact(project.project_root, "target", "manifest.json") + assert len(manifest["nodes"]) == 1 + + my_singular_test_node = manifest["nodes"]["test.test.my_singular_test"] + assert my_singular_test_node["description"] == "Some docs from a doc block" + assert my_singular_test_node["config"]["error_if"] == ">10" + assert my_singular_test_node["config"]["meta"] == {"some_key": "some_val"} + + +class TestPatchSingularTestInvalidName: + @pytest.fixture(scope="class") + def tests(self): + return { + "my_singular_test.sql": tests__my_singular_test_sql, + "schema_with_invalid_name.yml": tests__invalid_name_schema_yml, + } + + def test_compile(self, project): + _, log_output = run_dbt_and_capture(["compile"]) + + file_path = ( + "tests\\schema_with_invalid_name.yml" + if os.name == "nt" + else "tests/schema_with_invalid_name.yml" + ) + assert ( + f"Did not find matching node for patch with name 'my_double_test' in the 'data_tests' section of file '{file_path}'" + in log_output + ) diff --git a/tests/functional/defer_state/test_modified_state.py b/tests/functional/defer_state/test_modified_state.py index 2ded38e742b..2fe52584b51 100644 --- a/tests/functional/defer_state/test_modified_state.py +++ b/tests/functional/defer_state/test_modified_state.py @@ -1144,3 +1144,163 @@ def test_changed_semantic_model_contents(self, project): write_file(modified_semantic_model_schema_yml, "models", "schema.yml") results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 1 + + +class TestModifiedVars(BaseModifiedState): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": { + "state_modified_compare_vars": True, + }, + "vars": {"my_var": 1}, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "model_with_var.sql": "select {{ var('my_var') }} as id", + } + + def test_changed_vars(self, project): + self.run_and_save_state() + + # No var change + assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"]) + assert not run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"]) + + # Modify var (my_var: 1 -> 2) + update_config_file({"vars": {"my_var": 2}}, "dbt_project.yml") + assert run_dbt(["list", "-s", "state:modified", "--state", "./state"]) == [ + "test.model_with_var" + ] + assert run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"]) == [ + "test.model_with_var" + ] + + # Reset dbt_project.yml + update_config_file({"vars": {"my_var": 1}}, "dbt_project.yml") + + # Modify var via --var CLI flag + assert not run_dbt( + ["list", "--vars", '{"my_var": 1}', "-s", "state:modified", "--state", "./state"] + ) + assert run_dbt( + ["list", "--vars", '{"my_var": 2}', "-s", "state:modified", "--state", "./state"] + ) == ["test.model_with_var"] + + +macro_with_var_sql = """ +{% macro macro_with_var() %} + {{ var('my_var') }} +{% endmacro %} +""" + + +class TestModifiedMacroVars(BaseModifiedState): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": { + "state_modified_compare_vars": True, + }, + "vars": {"my_var": 1}, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "model_with_macro.sql": "select {{ macro_with_var() }} as id", + } + + @pytest.fixture(scope="class") + def macros(self): + return { + "macro_with_var.sql": macro_with_var_sql, + } + + def test_changed_vars(self, project): + self.run_and_save_state() + + # No var change + assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"]) + assert not run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"]) + + # Modify var (my_var: 1 -> 2) + update_config_file({"vars": {"my_var": 2}}, "dbt_project.yml") + assert run_dbt(["list", "-s", "state:modified", "--state", "./state"]) == [ + "test.model_with_macro" + ] + assert run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"]) == [ + "test.model_with_macro" + ] + + # Macros themselves not captured as modified because the var value depends on a node's context + assert not run_dbt(["list", "-s", "state:modified.macros", "--state", "./state"]) + + +# TODO: test versioned models, tests +model_with_var_schema_yml = """ +version: 2 +models: + - name: model_with_var + config: + materialized: "{{ var('my_var') }}" + +exposures: + - name: exposure_name + type: dashboard + owner: + name: "{{ var('my_var') }}" + +sources: + - name: jaffle_shop + database: "{{ var('my_var') }}" + schema: jaffle_shop + tables: + - name: orders + - name: customers +""" + + +class TestModifiedVarsSchemaYml(BaseModifiedState): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": { + "state_modified_compare_vars": True, + }, + "vars": {"my_var": "table"}, + } + + @pytest.fixture(scope="class") + def models(self): + return {"model_with_var.sql": "select 1 as id", "schema.yml": model_with_var_schema_yml} + + def test_changed_vars(self, project): + self.run_and_save_state() + + # No var change + assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"]) + assert not run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"]) + + # Modify var (my_var: table -> view) + update_config_file({"vars": {"my_var": "view"}}, "dbt_project.yml") + assert sorted(run_dbt(["list", "-s", "state:modified", "--state", "./state"])) == sorted( + [ + "test.model_with_var", + "exposure:test.exposure_name", + "source:test.jaffle_shop.customers", + "source:test.jaffle_shop.orders", + ] + ) + assert sorted( + run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"]) + ) == sorted( + [ + "test.model_with_var", + "exposure:test.exposure_name", + "source:test.jaffle_shop.customers", + "source:test.jaffle_shop.orders", + ] + ) diff --git a/tests/functional/list/test_list.py b/tests/functional/list/test_list.py index 6894ec96bb4..7b98e5c8251 100644 --- a/tests/functional/list/test_list.py +++ b/tests/functional/list/test_list.py @@ -49,7 +49,10 @@ def expect_snapshot_output(self, happy_path_project): # noqa: F811 "json": { "name": "my_snapshot", "package_name": "test", - "depends_on": {"nodes": [], "macros": []}, + "depends_on": { + "nodes": [], + "macros": [], + }, "tags": [], "config": { "enabled": True, diff --git a/tests/unit/config/test_project.py b/tests/unit/config/test_project.py index ab842c164d7..ddd519cc6ee 100644 --- a/tests/unit/config/test_project.py +++ b/tests/unit/config/test_project.py @@ -31,7 +31,7 @@ class TestProjectMethods: def test_all_source_paths(self, project: Project): assert ( project.all_source_paths.sort() - == ["models", "seeds", "snapshots", "analyses", "macros"].sort() + == ["models", "seeds", "snapshots", "analyses", "macros", "tests"].sort() ) def test_generic_test_paths(self, project: Project): @@ -99,7 +99,8 @@ def test_defaults(self): self.assertEqual(project.test_paths, ["tests"]) self.assertEqual(project.analysis_paths, ["analyses"]) self.assertEqual( - set(project.docs_paths), set(["models", "seeds", "snapshots", "analyses", "macros"]) + set(project.docs_paths), + {"models", "seeds", "snapshots", "analyses", "macros", "tests"}, ) self.assertEqual(project.asset_paths, []) self.assertEqual(project.target_path, "target") @@ -128,7 +129,7 @@ def test_implicit_overrides(self): ) self.assertEqual( set(project.docs_paths), - set(["other-models", "seeds", "snapshots", "analyses", "macros"]), + {"other-models", "seeds", "snapshots", "analyses", "macros", "tests"}, ) def test_all_overrides(self): diff --git a/tests/unit/config/test_runtime.py b/tests/unit/config/test_runtime.py index 816ec8f98c3..d03d33dab94 100644 --- a/tests/unit/config/test_runtime.py +++ b/tests/unit/config/test_runtime.py @@ -129,7 +129,7 @@ def test_from_args(self): self.assertEqual(config.test_paths, ["tests"]) self.assertEqual(config.analysis_paths, ["analyses"]) self.assertEqual( - set(config.docs_paths), set(["models", "seeds", "snapshots", "analyses", "macros"]) + set(config.docs_paths), {"models", "seeds", "snapshots", "analyses", "macros", "tests"} ) self.assertEqual(config.asset_paths, []) self.assertEqual(config.target_path, "target") diff --git a/tests/unit/contracts/graph/test_manifest.py b/tests/unit/contracts/graph/test_manifest.py index 36f14537b09..fd5e7cf9a55 100644 --- a/tests/unit/contracts/graph/test_manifest.py +++ b/tests/unit/contracts/graph/test_manifest.py @@ -96,6 +96,7 @@ "deprecation_date", "defer_relation", "time_spine", + "vars", "batches", } ) diff --git a/tests/unit/contracts/graph/test_nodes.py b/tests/unit/contracts/graph/test_nodes.py index 4e3a2353105..69e2b10454a 100644 --- a/tests/unit/contracts/graph/test_nodes.py +++ b/tests/unit/contracts/graph/test_nodes.py @@ -1,3 +1,4 @@ +from argparse import Namespace import pickle import re from dataclasses import replace @@ -24,6 +25,12 @@ ) +@pytest.fixture +def args_for_flags() -> Namespace: + return Namespace( + state_modified_compare_vars=False + ) + def norm_whitespace(string): _RE_COMBINE_WHITESPACE = re.compile(r"\s+") string = _RE_COMBINE_WHITESPACE.sub(" ", string).strip() @@ -199,6 +206,7 @@ def basic_compiled_dict(): "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, "access": "protected", @@ -519,6 +527,7 @@ def basic_compiled_schema_test_dict(): "unrendered_config": { "severity": "warn", }, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } diff --git a/tests/unit/contracts/graph/test_nodes_parsed.py b/tests/unit/contracts/graph/test_nodes_parsed.py index eeec787dd54..3488e577290 100644 --- a/tests/unit/contracts/graph/test_nodes_parsed.py +++ b/tests/unit/contracts/graph/test_nodes_parsed.py @@ -65,8 +65,10 @@ @pytest.fixture -def flags_for_args() -> Namespace: - return Namespace(SEND_ANONYMOUS_USAGE_STATS=False) +def args_for_flags() -> Namespace: + return Namespace( + send_anonymous_usage_stats=False, state_modified_compare_vars=False + ) @pytest.fixture @@ -199,6 +201,7 @@ def base_parsed_model_dict(): "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, "access": AccessType.Protected.value, @@ -256,6 +259,7 @@ def minimal_parsed_model_dict(): "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, + "vars": {}, } @@ -322,6 +326,7 @@ def complex_parsed_model_dict(): "materialized": "ephemeral", "post_hook": ['insert into blah(a, b) select "1", 1'], }, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, "access": AccessType.Protected.value, @@ -532,6 +537,7 @@ def basic_parsed_seed_dict(): "meta": {}, "checksum": {"name": "path", "checksum": "seeds/seed.csv"}, "unrendered_config": {}, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @@ -638,6 +644,7 @@ def complex_parsed_seed_dict(): "unrendered_config": { "persist_docs": {"relation": True, "columns": True}, }, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @@ -838,6 +845,7 @@ def base_parsed_hook_dict(): "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @@ -932,6 +940,7 @@ def complex_parsed_hook_dict(): "column_types": {"a": "text"}, "materialized": "table", }, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @@ -1077,6 +1086,7 @@ def basic_parsed_schema_test_dict(): "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @@ -1166,6 +1176,7 @@ def complex_parsed_schema_test_dict(): "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {"materialized": "table", "severity": "WARN"}, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @@ -1561,6 +1572,7 @@ def basic_timestamp_snapshot_dict(): "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", }, + "vars": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @@ -1668,6 +1680,7 @@ def basic_check_snapshot_dict(): }, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "vars": {}, } @@ -1877,6 +1890,7 @@ def basic_parsed_source_definition_dict(): "enabled": True, }, "unrendered_config": {}, + "vars": {}, } @@ -1909,6 +1923,7 @@ def complex_parsed_source_definition_dict(): "freshness": {"warn_after": {"period": "hour", "count": 1}, "error_after": {}}, "loaded_at_field": "loaded_at", "unrendered_config": {}, + "vars": {}, } @@ -2080,6 +2095,7 @@ def basic_parsed_exposure_dict(): "enabled": True, }, "unrendered_config": {}, + "vars": {}, } @@ -2135,6 +2151,7 @@ def complex_parsed_exposure_dict(): "enabled": True, }, "unrendered_config": {}, + "vars": {}, } diff --git a/tests/unit/graph/test_selector_methods.py b/tests/unit/graph/test_selector_methods.py index 04aebe052d1..d50f0e2d9e0 100644 --- a/tests/unit/graph/test_selector_methods.py +++ b/tests/unit/graph/test_selector_methods.py @@ -1,3 +1,4 @@ +from argparse import Namespace import copy from dataclasses import replace from pathlib import Path @@ -643,6 +644,11 @@ def previous_state(manifest): return create_previous_state(manifest) +@pytest.fixture +def args_for_flags(): + return Namespace(state_modified_compare_vars=False) + + def add_node(manifest, node): manifest.nodes[node.unique_id] = node