From 0aa68b28b1dce824567749953dbc3f8a85f73894 Mon Sep 17 00:00:00 2001 From: Devon Fulcher <24593113+DevonFulcher@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:50:52 -0600 Subject: [PATCH] Consolidated meta config to use the same models (#361) ### Description This PR addresses [this](https://github.com/dbt-labs/dbt-semantic-interfaces/pull/358#discussion_r1813382960) feedback to consolidate the meta config into central models. This is technically a breaking change because some classes have been deleted, but Courtney confirmed that these classes are not being used by dbt-core. ### Checklist - [x] I have read [the contributing guide](https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/CONTRIBUTING.md) and understand what's expected of me - [x] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements) - [x] This PR includes tests, or tests are not required/relevant for this PR - [x] I have run `changie new` to [create a changelog entry](https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/CONTRIBUTING.md#adding-a-changelog-entry) --- .../Breaking Changes-20241024-131227.yaml | 6 ++ .../implementations/metric.py | 17 ++-- .../implementations/semantic_model.py | 16 ++-- .../parsing/dir_to_model.py | 23 +++++- dbt_semantic_interfaces/protocols/__init__.py | 2 - dbt_semantic_interfaces/protocols/metric.py | 15 +--- .../protocols/semantic_model.py | 15 +--- pyproject.toml | 2 +- tests/parsing/test_semantic_model_parsing.py | 82 +++++++++++++++++++ tests/test_implements_satisfy_protocols.py | 3 +- 10 files changed, 128 insertions(+), 53 deletions(-) create mode 100644 .changes/unreleased/Breaking Changes-20241024-131227.yaml diff --git a/.changes/unreleased/Breaking Changes-20241024-131227.yaml b/.changes/unreleased/Breaking Changes-20241024-131227.yaml new file mode 100644 index 00000000..8f84c1b5 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241024-131227.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Consolidated meta config to use the same models +time: 2024-10-24T13:12:27.343387-05:00 +custom: + Author: DevonFulcher + Issue: None diff --git a/dbt_semantic_interfaces/implementations/metric.py b/dbt_semantic_interfaces/implementations/metric.py index 204337a7..4b596f0a 100644 --- a/dbt_semantic_interfaces/implementations/metric.py +++ b/dbt_semantic_interfaces/implementations/metric.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence, Set +from typing import Dict, List, Optional, Sequence, Set from typing_extensions import override @@ -12,11 +12,14 @@ PydanticCustomInputParser, PydanticParseableValueType, ) +from dbt_semantic_interfaces.implementations.element_config import ( + PydanticSemanticLayerElementConfig, +) from dbt_semantic_interfaces.implementations.filters.where_filter import ( PydanticWhereFilterIntersection, ) from dbt_semantic_interfaces.implementations.metadata import PydanticMetadata -from dbt_semantic_interfaces.protocols import Metric, MetricConfig, ProtocolHint +from dbt_semantic_interfaces.protocols import Metric, ProtocolHint from dbt_semantic_interfaces.references import MeasureReference, MetricReference from dbt_semantic_interfaces.type_enums import ( ConversionCalculationType, @@ -202,14 +205,6 @@ class PydanticMetricTypeParams(HashableBaseModel): input_measures: List[PydanticMetricInputMeasure] = Field(default_factory=list) -class PydanticMetricConfig(HashableBaseModel, ProtocolHint[MetricConfig]): # noqa: D - @override - def _implements_protocol(self) -> MetricConfig: # noqa: D - return self - - meta: Dict[str, Any] = Field(default_factory=dict) - - class PydanticMetric(HashableBaseModel, ModelWithMetadataParsing, ProtocolHint[Metric]): """Describes a metric.""" @@ -224,7 +219,7 @@ def _implements_protocol(self) -> Metric: # noqa: D filter: Optional[PydanticWhereFilterIntersection] metadata: Optional[PydanticMetadata] label: Optional[str] = None - config: Optional[PydanticMetricConfig] + config: Optional[PydanticSemanticLayerElementConfig] time_granularity: Optional[str] = None @property diff --git a/dbt_semantic_interfaces/implementations/semantic_model.py b/dbt_semantic_interfaces/implementations/semantic_model.py index eca8d6fd..36ad145b 100644 --- a/dbt_semantic_interfaces/implementations/semantic_model.py +++ b/dbt_semantic_interfaces/implementations/semantic_model.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence +from typing import List, Optional, Sequence from typing_extensions import override @@ -8,6 +8,9 @@ HashableBaseModel, ModelWithMetadataParsing, ) +from dbt_semantic_interfaces.implementations.element_config import ( + PydanticSemanticLayerElementConfig, +) from dbt_semantic_interfaces.implementations.elements.dimension import PydanticDimension from dbt_semantic_interfaces.implementations.elements.entity import PydanticEntity from dbt_semantic_interfaces.implementations.elements.measure import PydanticMeasure @@ -16,7 +19,6 @@ from dbt_semantic_interfaces.protocols import ( ProtocolHint, SemanticModel, - SemanticModelConfig, SemanticModelDefaults, ) from dbt_semantic_interfaces.references import ( @@ -38,14 +40,6 @@ def _implements_protocol(self) -> SemanticModelDefaults: # noqa: D agg_time_dimension: Optional[str] -class PydanticSemanticModelConfig(HashableBaseModel, ProtocolHint[SemanticModelConfig]): # noqa: D - @override - def _implements_protocol(self) -> SemanticModelConfig: # noqa: D - return self - - meta: Dict[str, Any] = Field(default_factory=dict) - - class PydanticSemanticModel(HashableBaseModel, ModelWithMetadataParsing, ProtocolHint[SemanticModel]): """Describes a semantic model.""" @@ -65,7 +59,7 @@ def _implements_protocol(self) -> SemanticModel: label: Optional[str] = None metadata: Optional[PydanticMetadata] - config: Optional[PydanticSemanticModelConfig] + config: Optional[PydanticSemanticLayerElementConfig] @property def entity_references(self) -> List[LinkableElementReference]: # noqa: D diff --git a/dbt_semantic_interfaces/parsing/dir_to_model.py b/dbt_semantic_interfaces/parsing/dir_to_model.py index 418ca254..c347b8a4 100644 --- a/dbt_semantic_interfaces/parsing/dir_to_model.py +++ b/dbt_semantic_interfaces/parsing/dir_to_model.py @@ -3,11 +3,17 @@ import traceback from dataclasses import dataclass from string import Template -from typing import Dict, List, Optional, Type, Union +from typing import Dict, List, Optional, Sequence, Type, Union from jsonschema import exceptions from dbt_semantic_interfaces.errors import ParsingException +from dbt_semantic_interfaces.implementations.element_config import ( + PydanticSemanticLayerElementConfig, +) +from dbt_semantic_interfaces.implementations.elements.dimension import PydanticDimension +from dbt_semantic_interfaces.implementations.elements.entity import PydanticEntity +from dbt_semantic_interfaces.implementations.elements.measure import PydanticMeasure from dbt_semantic_interfaces.implementations.metric import PydanticMetric from dbt_semantic_interfaces.implementations.project_configuration import ( PydanticProjectConfiguration, @@ -334,7 +340,20 @@ def parse_config_yaml( results.append(metric_class.parse_obj(object_cfg)) elif document_type == SEMANTIC_MODEL_TYPE: semantic_model_validator.validate(config_document[document_type]) - results.append(semantic_model_class.parse_obj(object_cfg)) + sm = semantic_model_class.parse_obj(object_cfg) + # Combine configs according to the behavior documented here https://docs.getdbt.com/reference/configs-and-properties#combining-configs + elements: Sequence[Union[PydanticDimension, PydanticEntity, PydanticMeasure]] = [ + *sm.dimensions, + *sm.entities, + *sm.measures, + ] + for element in elements: + if sm.config is not None: + if element.config is None: + element.config = PydanticSemanticLayerElementConfig(meta=sm.config.meta) + else: + element.config.meta = {**sm.config.meta, **element.config.meta} + results.append(sm) elif document_type == PROJECT_CONFIGURATION_TYPE: project_configuration_validator.validate(config_document[document_type]) results.append(project_configuration_class.parse_obj(object_cfg)) diff --git a/dbt_semantic_interfaces/protocols/__init__.py b/dbt_semantic_interfaces/protocols/__init__.py index 76414058..f9dbea5b 100644 --- a/dbt_semantic_interfaces/protocols/__init__.py +++ b/dbt_semantic_interfaces/protocols/__init__.py @@ -14,7 +14,6 @@ ConstantPropertyInput, ConversionTypeParams, Metric, - MetricConfig, MetricInput, MetricInputMeasure, MetricTimeWindow, @@ -28,7 +27,6 @@ ) from dbt_semantic_interfaces.protocols.semantic_model import ( # noqa:F401 SemanticModel, - SemanticModelConfig, SemanticModelDefaults, SemanticModelT, ) diff --git a/dbt_semantic_interfaces/protocols/metric.py b/dbt_semantic_interfaces/protocols/metric.py index a53d4102..f5cc384b 100644 --- a/dbt_semantic_interfaces/protocols/metric.py +++ b/dbt_semantic_interfaces/protocols/metric.py @@ -1,8 +1,9 @@ from __future__ import annotations from abc import abstractmethod -from typing import Any, Dict, Optional, Protocol, Sequence +from typing import Optional, Protocol, Sequence +from dbt_semantic_interfaces.protocols.meta import SemanticLayerElementConfig from dbt_semantic_interfaces.protocols.metadata import Metadata from dbt_semantic_interfaces.protocols.where_filter import WhereFilterIntersection from dbt_semantic_interfaces.references import MeasureReference, MetricReference @@ -263,16 +264,6 @@ def cumulative_type_params(self) -> Optional[CumulativeTypeParams]: # noqa: D pass -class MetricConfig(Protocol): # noqa: D - """The config property allows you to configure additional resources/metadata.""" - - @property - @abstractmethod - def meta(self) -> Dict[str, Any]: - """The meta field can be used to set metadata for a resource.""" - pass - - class Metric(Protocol): """Describes a metric.""" @@ -327,7 +318,7 @@ def metadata(self) -> Optional[Metadata]: # noqa: D @property @abstractmethod - def config(self) -> Optional[MetricConfig]: # noqa: D + def config(self) -> Optional[SemanticLayerElementConfig]: # noqa: D pass @property diff --git a/dbt_semantic_interfaces/protocols/semantic_model.py b/dbt_semantic_interfaces/protocols/semantic_model.py index d253b39a..15af3634 100644 --- a/dbt_semantic_interfaces/protocols/semantic_model.py +++ b/dbt_semantic_interfaces/protocols/semantic_model.py @@ -1,11 +1,12 @@ from __future__ import annotations from abc import abstractmethod -from typing import Any, Dict, Optional, Protocol, Sequence, TypeVar +from typing import Optional, Protocol, Sequence, TypeVar from dbt_semantic_interfaces.protocols.dimension import Dimension from dbt_semantic_interfaces.protocols.entity import Entity from dbt_semantic_interfaces.protocols.measure import Measure +from dbt_semantic_interfaces.protocols.meta import SemanticLayerElementConfig from dbt_semantic_interfaces.protocols.metadata import Metadata from dbt_semantic_interfaces.protocols.node_relation import NodeRelation from dbt_semantic_interfaces.references import ( @@ -27,16 +28,6 @@ def agg_time_dimension(self) -> Optional[str]: pass -class SemanticModelConfig(Protocol): # noqa: D - """The config property allows you to configure additional resources/metadata.""" - - @property - @abstractmethod - def meta(self) -> Dict[str, Any]: - """The meta field can be used to set metadata for a resource.""" - pass - - class SemanticModel(Protocol): """Describes a semantic model.""" @@ -148,7 +139,7 @@ def metadata(self) -> Optional[Metadata]: # noqa: D @property @abstractmethod - def config(self) -> Optional[SemanticModelConfig]: # noqa: D + def config(self) -> Optional[SemanticLayerElementConfig]: # noqa: D pass @abstractmethod diff --git a/pyproject.toml b/pyproject.toml index 1f3090fe..cfdd39b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dbt-semantic-interfaces" -version = "0.7.4" +version = "0.8.0" description = 'The shared semantic layer definitions that dbt-core and MetricFlow use' readme = "README.md" requires-python = ">=3.8" diff --git a/tests/parsing/test_semantic_model_parsing.py b/tests/parsing/test_semantic_model_parsing.py index 34ad851f..61839360 100644 --- a/tests/parsing/test_semantic_model_parsing.py +++ b/tests/parsing/test_semantic_model_parsing.py @@ -538,3 +538,85 @@ def test_semantic_model_dimension_validity_params_parsing() -> None: assert end_dimension.type_params.validity_params is not None assert end_dimension.type_params.validity_params.is_start is False assert end_dimension.type_params.validity_params.is_end is True + + +def test_semantic_model_element_config_merging() -> None: + """Test for merging element config metadata from semantic model into dimension, entity, and measure objects.""" + yaml_contents = textwrap.dedent( + """\ + semantic_model: + name: sm + config: + meta: + sm_metadata: asdf + node_relation: + alias: source_table + schema_name: some_schema + dimensions: + - name: dim_0 + type: time + type_params: + time_granularity: day + config: + meta: + sm_metadata: qwer + dim_metadata: fdsa + - name: dim_1 + type: time + type_params: + time_granularity: day + config: + meta: + dim_metadata: mlkj + sm_metadata: zxcv + - name: dim_2 + type: time + type_params: + time_granularity: day + - name: dim_3 + type: time + type_params: + time_granularity: day + config: + meta: + dim_metadata: gfds + entities: + - name: entity_0 + type: primary + config: + meta: + sm_metadata: hjkl + measures: + - name: measure_0 + agg: count_distinct + config: + meta: + sm_metadata: ijkl + """ + ) + file = YamlConfigFile(filepath="test_dir/inline_for_test", contents=yaml_contents) + + build_result = parse_yaml_files_to_semantic_manifest(files=[file, EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE]) + + assert len(build_result.semantic_manifest.semantic_models) == 1 + semantic_model = build_result.semantic_manifest.semantic_models[0] + assert semantic_model.config is not None + assert semantic_model.config.meta["sm_metadata"] == "asdf" + assert len(semantic_model.dimensions) == 4 + assert semantic_model.dimensions[0].config is not None + assert semantic_model.dimensions[0].config.meta["sm_metadata"] == "qwer" + assert semantic_model.dimensions[0].config.meta["dim_metadata"] == "fdsa" + assert semantic_model.dimensions[1].config is not None + assert semantic_model.dimensions[1].config.meta["sm_metadata"] == "zxcv" + assert semantic_model.dimensions[1].config.meta["dim_metadata"] == "mlkj" + assert semantic_model.dimensions[2].config is not None + assert semantic_model.dimensions[2].config.meta["sm_metadata"] == "asdf" + assert semantic_model.dimensions[3].config is not None + assert semantic_model.dimensions[3].config.meta["dim_metadata"] == "gfds" + assert semantic_model.dimensions[3].config.meta["sm_metadata"] == "asdf" + assert len(semantic_model.entities) == 1 + assert semantic_model.entities[0].config is not None + assert semantic_model.entities[0].config.meta["sm_metadata"] == "hjkl" + assert len(semantic_model.measures) == 1 + assert semantic_model.measures[0].config is not None + assert semantic_model.measures[0].config.meta["sm_metadata"] == "ijkl" diff --git a/tests/test_implements_satisfy_protocols.py b/tests/test_implements_satisfy_protocols.py index 0c4ec8e7..5919c83f 100644 --- a/tests/test_implements_satisfy_protocols.py +++ b/tests/test_implements_satisfy_protocols.py @@ -25,7 +25,6 @@ from dbt_semantic_interfaces.implementations.metric import ( PydanticConversionTypeParams, PydanticMetric, - PydanticMetricConfig, PydanticMetricInput, PydanticMetricInputMeasure, PydanticMetricTypeParams, @@ -128,7 +127,7 @@ filter=builds(PydanticWhereFilter) | none(), metadata=OPTIONAL_METADATA_STRATEGY, label=OPTIONAL_STR_STRATEGY, - config=builds(PydanticMetricConfig), + config=builds(PydanticSemanticLayerElementConfig), ) SAVED_QUERY_STRATEGY = builds(