From 57fd0fa36bbd99af9b1741986915c75763f8b531 Mon Sep 17 00:00:00 2001 From: Will Deng Date: Tue, 12 Nov 2024 16:30:05 -0500 Subject: [PATCH 1/8] Upgrade DSI to 0.8.1 and fix class changes --- core/dbt/artifacts/resources/v1/metric.py | 12 +- .../resources/v1/semantic_layer_components.py | 14 +- core/dbt/contracts/graph/unparsed.py | 2 +- core/dbt/parser/schema_yaml_readers.py | 43 +-- core/setup.py | 2 +- schemas/dbt/manifest/v12.json | 280 ++---------------- 6 files changed, 51 insertions(+), 302 deletions(-) diff --git a/core/dbt/artifacts/resources/v1/metric.py b/core/dbt/artifacts/resources/v1/metric.py index 0c6da764220..0924d4d12af 100644 --- a/core/dbt/artifacts/resources/v1/metric.py +++ b/core/dbt/artifacts/resources/v1/metric.py @@ -46,7 +46,7 @@ def post_aggregation_measure_reference(self) -> MeasureReference: @dataclass class MetricTimeWindow(dbtClassMixin): count: int - granularity: TimeGranularity + granularity: str @dataclass @@ -55,7 +55,7 @@ class MetricInput(dbtClassMixin): filter: Optional[WhereFilterIntersection] = None alias: Optional[str] = None offset_window: Optional[MetricTimeWindow] = None - offset_to_grain: Optional[TimeGranularity] = None + offset_to_grain: Optional[str] = None def as_reference(self) -> MetricReference: return MetricReference(element_name=self.name) @@ -83,7 +83,7 @@ class ConversionTypeParams(dbtClassMixin): @dataclass class CumulativeTypeParams(dbtClassMixin): window: Optional[MetricTimeWindow] = None - grain_to_date: Optional[TimeGranularity] = None + grain_to_date: Optional[str] = None period_agg: PeriodAggregation = PeriodAggregation.FIRST @@ -95,7 +95,9 @@ class MetricTypeParams(dbtClassMixin): denominator: Optional[MetricInput] = None expr: Optional[str] = None window: Optional[MetricTimeWindow] = None - grain_to_date: Optional[TimeGranularity] = None + grain_to_date: Optional[TimeGranularity] = ( + None # legacy, use cumulative_type_params.grain_to_date + ) metrics: Optional[List[MetricInput]] = None conversion_type_params: Optional[ConversionTypeParams] = None cumulative_type_params: Optional[CumulativeTypeParams] = None @@ -121,7 +123,7 @@ class Metric(GraphResource): type_params: MetricTypeParams filter: Optional[WhereFilterIntersection] = None metadata: Optional[SourceFileMetadata] = None - time_granularity: Optional[TimeGranularity] = None + time_granularity: Optional[str] = None resource_type: Literal[NodeType.Metric] meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta()) tags: List[str] = field(default_factory=list) diff --git a/core/dbt/artifacts/resources/v1/semantic_layer_components.py b/core/dbt/artifacts/resources/v1/semantic_layer_components.py index 89f87512b0b..65fb8ce0de7 100644 --- a/core/dbt/artifacts/resources/v1/semantic_layer_components.py +++ b/core/dbt/artifacts/resources/v1/semantic_layer_components.py @@ -12,17 +12,21 @@ class WhereFilter(dbtClassMixin): where_sql_template: str - @property - def call_parameter_sets(self) -> FilterCallParameterSets: - return WhereFilterParser.parse_call_parameter_sets(self.where_sql_template) + def call_parameter_sets( + self, custom_granularity_names: Sequence[str] + ) -> FilterCallParameterSets: + return WhereFilterParser.parse_call_parameter_sets( + self.where_sql_template, custom_granularity_names=custom_granularity_names + ) @dataclass class WhereFilterIntersection(dbtClassMixin): where_filters: List[WhereFilter] - @property - def filter_expression_parameter_sets(self) -> Sequence[Tuple[str, FilterCallParameterSets]]: + def filter_expression_parameter_sets( + self, custom_granularity_names: Sequence[str] + ) -> Sequence[Tuple[str, FilterCallParameterSets]]: raise NotImplementedError diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 5e63c487e89..f78ba15a50f 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -564,7 +564,7 @@ class UnparsedMetricInput(dbtClassMixin): filter: Union[str, List[str], None] = None alias: Optional[str] = None offset_window: Optional[str] = None - offset_to_grain: Optional[str] = None # str is really a TimeGranularity Enum + offset_to_grain: Optional[str] = None @dataclass diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 9b4a550b5d3..6e312780141 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -228,19 +228,11 @@ def _get_input_measures( def _get_period_agg(self, unparsed_period_agg: str) -> PeriodAggregation: return PeriodAggregation(unparsed_period_agg) - def _get_optional_grain_to_date( - self, unparsed_grain_to_date: Optional[str] - ) -> Optional[TimeGranularity]: - if not unparsed_grain_to_date: - return None - - return TimeGranularity(unparsed_grain_to_date) - def _get_optional_time_window( self, unparsed_window: Optional[str] ) -> Optional[MetricTimeWindow]: if unparsed_window is not None: - parts = unparsed_window.split(" ") + parts = unparsed_window.lower().split(" ") if len(parts) != 2: raise YamlParseDictError( self.yaml.path, @@ -252,16 +244,11 @@ def _get_optional_time_window( granularity = parts[1] # once we drop python 3.8 this could just be `granularity = parts[0].removesuffix('s') - if granularity.endswith("s"): - # months -> month + if granularity.endswith("s") and granularity[:-1] in [ + item.value for item in TimeGranularity + ]: + # Can only remove the `s` if it's a standard grain, months -> month granularity = granularity[:-1] - if granularity not in [item.value for item in TimeGranularity]: - raise YamlParseDictError( - self.yaml.path, - "window", - {"window": unparsed_window}, - f"Invalid time granularity {granularity} in cumulative/conversion metric window string: ({unparsed_window})", - ) count = parts[0] if not count.isdigit(): @@ -274,7 +261,7 @@ def _get_optional_time_window( return MetricTimeWindow( count=int(count), - granularity=TimeGranularity(granularity), + granularity=granularity, ) else: return None @@ -283,16 +270,12 @@ def _get_metric_input(self, unparsed: Union[UnparsedMetricInput, str]) -> Metric if isinstance(unparsed, str): return MetricInput(name=unparsed) else: - offset_to_grain: Optional[TimeGranularity] = None - if unparsed.offset_to_grain is not None: - offset_to_grain = TimeGranularity(unparsed.offset_to_grain) - return MetricInput( name=unparsed.name, filter=parse_where_filter(unparsed.filter), alias=unparsed.alias, offset_window=self._get_optional_time_window(unparsed.offset_window), - offset_to_grain=offset_to_grain, + offset_to_grain=unparsed.offset_to_grain, ) def _get_optional_metric_input( @@ -354,9 +337,7 @@ def _get_optional_cumulative_type_params( window=self._get_optional_time_window( unparsed_type_params.cumulative_type_params.window ), - grain_to_date=self._get_optional_grain_to_date( - unparsed_type_params.cumulative_type_params.grain_to_date - ), + grain_to_date=unparsed_type_params.cumulative_type_params.grain_to_date, period_agg=self._get_period_agg( unparsed_type_params.cumulative_type_params.period_agg ), @@ -369,6 +350,10 @@ def _get_metric_type_params(self, unparsed_metric: UnparsedMetric) -> MetricType grain_to_date: Optional[TimeGranularity] = None if type_params.grain_to_date is not None: + # This should've been changed to a string (to support custom grain), but since this + # is a legacy field waiting to be deprecated, we will not support custom grain here + # in order to force customers off of using this field. The field to use should be + # `cumulative_type_params.grain_to_date` grain_to_date = TimeGranularity(type_params.grain_to_date) return MetricTypeParams( @@ -435,9 +420,7 @@ def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] = label=unparsed.label, type=MetricType(unparsed.type), type_params=self._get_metric_type_params(unparsed), - time_granularity=( - TimeGranularity(unparsed.time_granularity) if unparsed.time_granularity else None - ), + time_granularity=unparsed.time_granularity, filter=parse_where_filter(unparsed.filter), meta=unparsed.meta, tags=unparsed.tags, diff --git a/core/setup.py b/core/setup.py index 7da702f0dda..ddd696c8160 100644 --- a/core/setup.py +++ b/core/setup.py @@ -69,7 +69,7 @@ # These are major-version-0 packages also maintained by dbt-labs. # Accept patches but avoid automatically updating past a set minor version range. "dbt-extractor>=0.5.0,<=0.6", - "dbt-semantic-interfaces>=0.7.4,<0.8", + "dbt-semantic-interfaces>=0.8.1,<0.9", # Minor versions for these are expected to be backwards-compatible "dbt-common>=1.13.0,<2.0", "dbt-adapters>=1.10.1,<2.0", diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index 892c0014e41..5e9198b07f6 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -8960,19 +8960,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -8990,19 +8978,7 @@ "offset_to_grain": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" @@ -9086,19 +9062,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -9116,19 +9080,7 @@ "offset_to_grain": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" @@ -9285,19 +9237,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -9315,19 +9255,7 @@ "offset_to_grain": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" @@ -9520,19 +9448,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -9605,19 +9521,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -9635,19 +9539,7 @@ "grain_to_date": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" @@ -9759,19 +9651,7 @@ "time_granularity": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" @@ -18671,19 +18551,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -18701,19 +18569,7 @@ "offset_to_grain": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" @@ -18797,19 +18653,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -18827,19 +18671,7 @@ "offset_to_grain": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" @@ -18996,19 +18828,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -19026,19 +18846,7 @@ "offset_to_grain": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" @@ -19231,19 +19039,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -19316,19 +19112,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -19346,19 +19130,7 @@ "grain_to_date": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" @@ -19470,19 +19242,7 @@ "time_granularity": { "anyOf": [ { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" }, { "type": "null" From 58794616c7cdd2943a07ac133c8a00bba03ea814 Mon Sep 17 00:00:00 2001 From: Will Deng Date: Tue, 12 Nov 2024 16:35:57 -0500 Subject: [PATCH 2/8] fix tests --- core/dbt/artifacts/resources/v1/metric.py | 8 ++++++++ core/dbt/artifacts/resources/v1/semantic_model.py | 11 +++++++++++ tests/functional/metrics/test_metrics.py | 12 ++++++------ tests/unit/contracts/graph/test_semantic_manifest.py | 12 ++++++------ .../test_semantic_layer_nodes_satisfy_protocols.py | 4 ++-- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/core/dbt/artifacts/resources/v1/metric.py b/core/dbt/artifacts/resources/v1/metric.py index 0924d4d12af..a025d39ae33 100644 --- a/core/dbt/artifacts/resources/v1/metric.py +++ b/core/dbt/artifacts/resources/v1/metric.py @@ -48,6 +48,14 @@ class MetricTimeWindow(dbtClassMixin): count: int granularity: str + @property + def window_string(self) -> str: # noqa: D + return f"{self.count} {self.granularity}" + + @property + def is_standard_granularity(self) -> bool: # noqa: D + return self.granularity.casefold() in {item.value.casefold() for item in TimeGranularity} + @dataclass class MetricInput(dbtClassMixin): diff --git a/core/dbt/artifacts/resources/v1/semantic_model.py b/core/dbt/artifacts/resources/v1/semantic_model.py index 3e021b2f469..1721633d639 100644 --- a/core/dbt/artifacts/resources/v1/semantic_model.py +++ b/core/dbt/artifacts/resources/v1/semantic_model.py @@ -31,6 +31,14 @@ """ +@dataclass +class SemanticLayerElementConfig(dbtClassMixin): + meta: Dict[str, Any] = field( + default_factory=dict, + metadata=MergeBehavior.Update.meta(), + ) + + @dataclass class Defaults(dbtClassMixin): agg_time_dimension: Optional[str] = None @@ -72,6 +80,7 @@ class Dimension(dbtClassMixin): type_params: Optional[DimensionTypeParams] = None expr: Optional[str] = None metadata: Optional[SourceFileMetadata] = None + config: Optional[SemanticLayerElementConfig] = None @property def reference(self) -> DimensionReference: @@ -106,6 +115,7 @@ class Entity(dbtClassMixin): label: Optional[str] = None role: Optional[str] = None expr: Optional[str] = None + config: Optional[SemanticLayerElementConfig] = None @property def reference(self) -> EntityReference: @@ -147,6 +157,7 @@ class Measure(dbtClassMixin): agg_params: Optional[MeasureAggregationParameters] = None non_additive_dimension: Optional[NonAdditiveDimension] = None agg_time_dimension: Optional[str] = None + config: Optional[SemanticLayerElementConfig] = None @property def reference(self) -> MeasureReference: diff --git a/tests/functional/metrics/test_metrics.py b/tests/functional/metrics/test_metrics.py index 3f8fba2a19c..7c0842a7414 100644 --- a/tests/functional/metrics/test_metrics.py +++ b/tests/functional/metrics/test_metrics.py @@ -85,7 +85,7 @@ def test_simple_metric( ) assert ( manifest.metrics["metric.test.number_of_people"].time_granularity - == TimeGranularity.MONTH + == TimeGranularity.MONTH.value ) assert manifest.metrics["metric.test.collective_tenure"].time_granularity is None @@ -436,25 +436,25 @@ def test_cumulative_metric(self, project): metric_ids = set(manifest.metrics.keys()) expected_metric_ids_to_cumulative_type_params = { "metric.test.weekly_visits": CumulativeTypeParams( - window=MetricTimeWindow(count=7, granularity=TimeGranularity.DAY), + window=MetricTimeWindow(count=7, granularity=TimeGranularity.DAY.value), period_agg=PeriodAggregation.AVERAGE, ), "metric.test.cumulative_orders": CumulativeTypeParams( period_agg=PeriodAggregation.LAST ), "metric.test.orders_ytd": CumulativeTypeParams( - grain_to_date=TimeGranularity.YEAR, period_agg=PeriodAggregation.FIRST + grain_to_date=TimeGranularity.YEAR.value, period_agg=PeriodAggregation.FIRST ), "metric.test.monthly_orders": CumulativeTypeParams( - window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH), + window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value), period_agg=PeriodAggregation.AVERAGE, ), "metric.test.yearly_orders": CumulativeTypeParams( - window=MetricTimeWindow(count=1, granularity=TimeGranularity.YEAR), + window=MetricTimeWindow(count=1, granularity=TimeGranularity.YEAR.value), period_agg=PeriodAggregation.FIRST, ), "metric.test.visits_mtd": CumulativeTypeParams( - grain_to_date=TimeGranularity.MONTH, period_agg=PeriodAggregation.FIRST + grain_to_date=TimeGranularity.MONTH.value, period_agg=PeriodAggregation.FIRST ), "metric.test.cumulative_visits": CumulativeTypeParams( period_agg=PeriodAggregation.FIRST diff --git a/tests/unit/contracts/graph/test_semantic_manifest.py b/tests/unit/contracts/graph/test_semantic_manifest.py index 043f9b7ad33..229fb314515 100644 --- a/tests/unit/contracts/graph/test_semantic_manifest.py +++ b/tests/unit/contracts/graph/test_semantic_manifest.py @@ -66,7 +66,7 @@ def test_require_yaml_configuration_for_mf_time_spines( ), ( MetricTypeParams( - window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH) + window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value) ), 1, False, @@ -75,7 +75,7 @@ def test_require_yaml_configuration_for_mf_time_spines( ( MetricTypeParams( cumulative_type_params=CumulativeTypeParams( - grain_to_date=TimeGranularity.MONTH, + grain_to_date=TimeGranularity.MONTH.value, ) ), 0, @@ -85,7 +85,7 @@ def test_require_yaml_configuration_for_mf_time_spines( ( MetricTypeParams( cumulative_type_params=CumulativeTypeParams( - window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH), + window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value), ) ), 0, @@ -100,7 +100,7 @@ def test_require_yaml_configuration_for_mf_time_spines( ), ( MetricTypeParams( - window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH) + window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value) ), 0, True, @@ -109,7 +109,7 @@ def test_require_yaml_configuration_for_mf_time_spines( ( MetricTypeParams( cumulative_type_params=CumulativeTypeParams( - grain_to_date=TimeGranularity.MONTH, + grain_to_date=TimeGranularity.MONTH.value, ) ), 0, @@ -119,7 +119,7 @@ def test_require_yaml_configuration_for_mf_time_spines( ( MetricTypeParams( cumulative_type_params=CumulativeTypeParams( - window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH), + window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value), ) ), 0, diff --git a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py index 3838b1a4a33..156b4f9b9a8 100644 --- a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py +++ b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py @@ -196,7 +196,7 @@ def where_filter() -> WhereFilter: @pytest.fixture(scope="session") def metric_time_window() -> MetricTimeWindow: - return MetricTimeWindow(count=1, granularity=TimeGranularity.DAY) + return MetricTimeWindow(count=1, granularity=TimeGranularity.DAY.value) @pytest.fixture(scope="session") @@ -211,7 +211,7 @@ def complex_metric_input(metric_time_window, where_filter) -> MetricInput: filter=where_filter, alias="aliased_metric_input", offset_window=metric_time_window, - offset_to_grain=TimeGranularity.DAY, + offset_to_grain=TimeGranularity.DAY.value, ) From 66fd87d6aeec35fd4e43e5d8d917be2a74a6ae97 Mon Sep 17 00:00:00 2001 From: Will Deng Date: Tue, 12 Nov 2024 18:45:42 -0500 Subject: [PATCH 3/8] add more tests --- tests/functional/metrics/fixtures.py | 36 ++++++++++++++++++++++++ tests/functional/metrics/test_metrics.py | 31 ++++++++++++++++---- tests/functional/time_spines/fixtures.py | 18 ++++++++++++ 3 files changed, 80 insertions(+), 5 deletions(-) diff --git a/tests/functional/metrics/fixtures.py b/tests/functional/metrics/fixtures.py index 5614fab8c63..bfd75fd4a2d 100644 --- a/tests/functional/metrics/fixtures.py +++ b/tests/functional/metrics/fixtures.py @@ -462,6 +462,8 @@ dimensions: - name: purchased_at type: TIME + type_params: + time_granularity: day entities: - name: purchase type: primary @@ -736,6 +738,8 @@ dimensions: - name: purchased_at type: TIME + type_params: + time_granularity: day entities: - name: purchase type: primary @@ -796,6 +800,20 @@ type: cumulative type_params: measure: num_visits + - name: visits_martian_day + label: Visits since start of martian_day + type: cumulative + type_params: + measure: num_visits + cumulative_type_params: + grain_to_date: martian_day + - name: visits_martian_day_window + label: Visits since start of martian_day window + type: cumulative + type_params: + measure: num_visits + cumulative_type_params: + window: 1 martian_day """ conversion_metric_yml = """ @@ -809,6 +827,24 @@ base_measure: num_visits conversion_measure: num_orders entity: purchase + - name: converted_orders_over_visits_with_window + label: Number of orders converted from visits with window + type: conversion + type_params: + conversion_type_params: + base_measure: num_visits + conversion_measure: num_orders + entity: purchase + window: 4 day + - name: converted_orders_over_visits_with_custom_window + label: Number of orders converted from visits with custom window + type: conversion + type_params: + conversion_type_params: + base_measure: num_visits + conversion_measure: num_orders + entity: purchase + window: 4 martian_day """ filtered_metrics_yml = """ diff --git a/tests/functional/metrics/test_metrics.py b/tests/functional/metrics/test_metrics.py index 7c0842a7414..9905cdfa385 100644 --- a/tests/functional/metrics/test_metrics.py +++ b/tests/functional/metrics/test_metrics.py @@ -34,6 +34,7 @@ semantic_model_people_yml, semantic_model_purchasing_yml, ) +from tests.functional.time_spines.fixtures import time_spine_yml class TestSimpleMetrics: @@ -361,6 +362,7 @@ def models(self): "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": conversion_semantic_model_purchasing_yml, "conversion_metric.yml": conversion_metric_yml, + "time_spine.yml": time_spine_yml, } @pytest.fixture(scope="class") @@ -382,10 +384,16 @@ def test_conversion_metric( # make sure the metric is in the manifest manifest = get_manifest(project.project_root) metric_ids = list(manifest.metrics.keys()) - expected_metric_ids = [ - "metric.test.converted_orders_over_visits", - ] - assert metric_ids == expected_metric_ids + expected_metric_ids = { + "metric.test.converted_orders_over_visits": None, + "metric.test.converted_orders_over_visits_with_window": MetricTimeWindow( + count=4, granularity=TimeGranularity.DAY.value + ), + "metric.test.converted_orders_over_visits_with_custom_window": MetricTimeWindow( + count=4, granularity="martian_day" + ), + } + assert set(metric_ids) == set(expected_metric_ids.keys()) assert manifest.metrics[ "metric.test.converted_orders_over_visits" ].type_params.conversion_type_params @@ -409,6 +417,14 @@ def test_conversion_metric( ].type_params.conversion_type_params.entity == "purchase" ) + for ( + metric_id, + expected_window, + ) in expected_metric_ids.items(): + assert ( + manifest.metrics[metric_id].type_params.conversion_type_params.window + == expected_window + ), f"Found unexpected conversion window for {metric_id}" class TestCumulativeMetric: @@ -418,7 +434,8 @@ def models(self): "purchasing.sql": purchasing_model_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": conversion_semantic_model_purchasing_yml, - "conversion_metric.yml": cumulative_metric_yml, + "cumulative_metric.yml": cumulative_metric_yml, + "time_spine.yml": time_spine_yml, } @pytest.fixture(scope="class") @@ -459,6 +476,10 @@ def test_cumulative_metric(self, project): "metric.test.cumulative_visits": CumulativeTypeParams( period_agg=PeriodAggregation.FIRST ), + "metric.test.visits_martian_day": CumulativeTypeParams(grain_to_date="martian_day"), + "metric.test.visits_martian_day_window": CumulativeTypeParams( + window=MetricTimeWindow(count=1, granularity="martian_day"), + ), } assert metric_ids == set(expected_metric_ids_to_cumulative_type_params.keys()) for ( diff --git a/tests/functional/time_spines/fixtures.py b/tests/functional/time_spines/fixtures.py index 0f67488ff11..e6012cb7154 100644 --- a/tests/functional/time_spines/fixtures.py +++ b/tests/functional/time_spines/fixtures.py @@ -103,3 +103,21 @@ - name: date_day granularity: day """ + +time_spine_yml = """ +version: 2 + +models: + - name: metricflow_time_spine + time_spine: + standard_granularity_column: date_day + custom_granularities: + - name: retail_month + - name: martian_day + column_name: martian_day + columns: + - name: date_day + granularity: day + - name: retail_month + - name: martian_day +""" From 6fe69a0e2c0d6e340d8d71e95060cb1566849ec5 Mon Sep 17 00:00:00 2001 From: Will Deng Date: Tue, 12 Nov 2024 18:47:01 -0500 Subject: [PATCH 4/8] changelog --- .changes/unreleased/Dependencies-20241112-163815.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Dependencies-20241112-163815.yaml diff --git a/.changes/unreleased/Dependencies-20241112-163815.yaml b/.changes/unreleased/Dependencies-20241112-163815.yaml new file mode 100644 index 00000000000..eb21def4b13 --- /dev/null +++ b/.changes/unreleased/Dependencies-20241112-163815.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Upgrading dbt-semantic-interfaces to 0.8.1 for custom grain support +time: 2024-11-12T16:38:15.351519-05:00 +custom: + Author: WilliamDee + Issue: None From 7bf447ab434dfca9a65f53fd4726034541d65220 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Mon, 2 Dec 2024 17:41:35 -0500 Subject: [PATCH 5/8] Bump DSI to 0.8.2 --- core/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/setup.py b/core/setup.py index ddd696c8160..506115bc7fb 100644 --- a/core/setup.py +++ b/core/setup.py @@ -69,7 +69,7 @@ # These are major-version-0 packages also maintained by dbt-labs. # Accept patches but avoid automatically updating past a set minor version range. "dbt-extractor>=0.5.0,<=0.6", - "dbt-semantic-interfaces>=0.8.1,<0.9", + "dbt-semantic-interfaces>=0.8.2,<0.9", # Minor versions for these are expected to be backwards-compatible "dbt-common>=1.13.0,<2.0", "dbt-adapters>=1.10.1,<2.0", From 074375b1d4ad8f307e7d1cbaae26c62813f8f4aa Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Mon, 2 Dec 2024 23:36:54 -0500 Subject: [PATCH 6/8] Bump DSI to 0.8.3 --- core/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/setup.py b/core/setup.py index 506115bc7fb..be77d1ba73b 100644 --- a/core/setup.py +++ b/core/setup.py @@ -69,7 +69,7 @@ # These are major-version-0 packages also maintained by dbt-labs. # Accept patches but avoid automatically updating past a set minor version range. "dbt-extractor>=0.5.0,<=0.6", - "dbt-semantic-interfaces>=0.8.2,<0.9", + "dbt-semantic-interfaces>=0.8.3,<0.9", # Minor versions for these are expected to be backwards-compatible "dbt-common>=1.13.0,<2.0", "dbt-adapters>=1.10.1,<2.0", From 7aae48d38f41bf486cecd39165780c464cf1d412 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Mon, 2 Dec 2024 23:39:27 -0500 Subject: [PATCH 7/8] Update changelog --- .changes/unreleased/Dependencies-20241112-163815.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Dependencies-20241112-163815.yaml b/.changes/unreleased/Dependencies-20241112-163815.yaml index eb21def4b13..e89502bee78 100644 --- a/.changes/unreleased/Dependencies-20241112-163815.yaml +++ b/.changes/unreleased/Dependencies-20241112-163815.yaml @@ -1,5 +1,5 @@ kind: Dependencies -body: Upgrading dbt-semantic-interfaces to 0.8.1 for custom grain support +body: Upgrading dbt-semantic-interfaces to 0.8.3 for custom grain support in offset windows time: 2024-11-12T16:38:15.351519-05:00 custom: Author: WilliamDee From 363e44b79b5e28f80811f504f3655f135159ad9b Mon Sep 17 00:00:00 2001 From: Will Deng Date: Wed, 4 Dec 2024 11:28:15 -0500 Subject: [PATCH 8/8] Custom grain was decided to only be supported in input_metric.offset_window, so removing tests that includes them otherwise --- tests/functional/metrics/fixtures.py | 58 ++++++++++++++---------- tests/functional/metrics/test_metrics.py | 21 ++++++--- 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/tests/functional/metrics/fixtures.py b/tests/functional/metrics/fixtures.py index bfd75fd4a2d..693c8a3fc4d 100644 --- a/tests/functional/metrics/fixtures.py +++ b/tests/functional/metrics/fixtures.py @@ -496,6 +496,16 @@ name: sum_order_revenue denominator: name: count_orders + + - name: sum_order_revenue_plus_one_custom_offset_window + label: "Total order revenue, plus 1 with custom offset window" + description: "The total order revenue plus 1 offset by 1 martian day" + type: derived + type_params: + metrics: + - name: sum_order_revenue + offset_window: 1 martian_day + expr: "sum_order_revenue + 1" """ disabled_metric_level_schema_yml = """ @@ -800,20 +810,21 @@ type: cumulative type_params: measure: num_visits - - name: visits_martian_day - label: Visits since start of martian_day - type: cumulative - type_params: - measure: num_visits - cumulative_type_params: - grain_to_date: martian_day - - name: visits_martian_day_window - label: Visits since start of martian_day window - type: cumulative - type_params: - measure: num_visits - cumulative_type_params: - window: 1 martian_day + # TODO: Re-enable this when custom grain is supported for this type + # - name: visits_martian_day + # label: Visits since start of martian_day + # type: cumulative + # type_params: + # measure: num_visits + # cumulative_type_params: + # grain_to_date: martian_day + # - name: visits_martian_day_window + # label: Visits since start of martian_day window + # type: cumulative + # type_params: + # measure: num_visits + # cumulative_type_params: + # window: 1 martian_day """ conversion_metric_yml = """ @@ -836,15 +847,16 @@ conversion_measure: num_orders entity: purchase window: 4 day - - name: converted_orders_over_visits_with_custom_window - label: Number of orders converted from visits with custom window - type: conversion - type_params: - conversion_type_params: - base_measure: num_visits - conversion_measure: num_orders - entity: purchase - window: 4 martian_day + # TODO: Re-enable this when custom grain is supported for this type + # - name: converted_orders_over_visits_with_custom_window + # label: Number of orders converted from visits with custom window + # type: conversion + # type_params: + # conversion_type_params: + # base_measure: num_visits + # conversion_measure: num_orders + # entity: purchase + # window: 4 martian_day """ filtered_metrics_yml = """ diff --git a/tests/functional/metrics/test_metrics.py b/tests/functional/metrics/test_metrics.py index 9905cdfa385..eddf641801b 100644 --- a/tests/functional/metrics/test_metrics.py +++ b/tests/functional/metrics/test_metrics.py @@ -252,6 +252,7 @@ def models(self): "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": semantic_model_purchasing_yml, "derived_metric.yml": derived_metric_yml, + "time_spine.yml": time_spine_yml, } # not strictly necessary to use "real" mock data for this test @@ -277,6 +278,7 @@ def test_derived_metric( "metric.test.count_orders", "metric.test.sum_order_revenue", "metric.test.average_order_value", + "metric.test.sum_order_revenue_plus_one_custom_offset_window", ] assert metric_ids == expected_metric_ids @@ -298,12 +300,24 @@ def test_derived_metric( "metric.test.sum_order_revenue", ] + derived_metric_with_custom_offset_window = manifest.metrics[ + "metric.test.sum_order_revenue_plus_one_custom_offset_window" + ] + assert len(derived_metric_with_custom_offset_window.input_metrics) == 1 + assert derived_metric_with_custom_offset_window.input_metrics[ + 0 + ].offset_window == MetricTimeWindow(count=1, granularity="martian_day") + # actually compile results = run_dbt(["compile", "--select", "downstream_model"]) compiled_code = results[0].node.compiled_code # make sure all these metrics properties show up in compiled SQL for metric_name in manifest.metrics: + if metric_name == "metric.test.sum_order_revenue_plus_one_custom_offset_window": + # Skip this metric + continue + parsed_metric_node = manifest.metrics[metric_name] for property in [ "name", @@ -389,9 +403,6 @@ def test_conversion_metric( "metric.test.converted_orders_over_visits_with_window": MetricTimeWindow( count=4, granularity=TimeGranularity.DAY.value ), - "metric.test.converted_orders_over_visits_with_custom_window": MetricTimeWindow( - count=4, granularity="martian_day" - ), } assert set(metric_ids) == set(expected_metric_ids.keys()) assert manifest.metrics[ @@ -476,10 +487,6 @@ def test_cumulative_metric(self, project): "metric.test.cumulative_visits": CumulativeTypeParams( period_agg=PeriodAggregation.FIRST ), - "metric.test.visits_martian_day": CumulativeTypeParams(grain_to_date="martian_day"), - "metric.test.visits_martian_day_window": CumulativeTypeParams( - window=MetricTimeWindow(count=1, granularity="martian_day"), - ), } assert metric_ids == set(expected_metric_ids_to_cumulative_type_params.keys()) for (