Cleanup

.changes/unreleased/Features-20230911-190924.yaml

@@ -1,5 +1,5 @@
 kind: Features
-body: DatePart aggregation for time dimensions
+body: Enable DATE PART aggregation for time dimensions
 time: 2023-09-11T19:09:24.960342-07:00
 custom:
   Author: courtneyholcomb

metricflow/dataflow/builder/dataflow_plan_builder.py

@@ -792,7 +792,7 @@ def _build_aggregated_measures_from_measure_source_node(
         # If querying an offset metric, join to time spine.
         join_to_time_spine_node: Optional[JoinToTimeSpineNode] = None
         if metric_spec.offset_window or metric_spec.offset_to_grain:
-            assert metric_time_dimension_specs, "Joining to time spine requires querying with metric_time."
+            assert metric_time_dimension_specs, "Joining to time spine requires querying with metric time."
             join_to_time_spine_node = JoinToTimeSpineNode(
                 parent_node=time_range_node or measure_recipe.measure_node,
                 metric_time_dimension_specs=metric_time_dimension_specs,

metricflow/naming/linkable_spec_name.py

@@ -63,7 +63,10 @@ def from_name(qualified_name: str) -> StructuredLinkableSpecName:
 
     @property
     def qualified_name(self) -> str:
-        """Return the full name form. e.g. ds or listing__ds__month."""
+        """Return the full name form. e.g. ds or listing__ds__month.
+
+        If date_part is specified, don't include granularity in qualified_name since it will not impact the result.
+        """
         items = list(self.entity_link_names) + [self.element_name]
         if self.date_part:
             items.append(self.date_part_suffix(date_part=self.date_part))

metricflow/query/query_parser.py

@@ -702,7 +702,9 @@ def _parse_linkable_elements(
             for linkable_element in linkable_elements:
                 parsed_name = StructuredLinkableSpecName.from_name(linkable_element.name)
                 if parsed_name.time_granularity:
-                    raise ValueError("Time granularity must be passed in the grain attribute for group_by query param.")
+                    raise ValueError(
+                        "Time granularity must be passed in the `grain` attribute for `group_by` query param."
+                    )
                 structured_name = StructuredLinkableSpecName(
                     entity_link_names=parsed_name.entity_link_names,
                     element_name=parsed_name.element_name,
@@ -721,15 +723,15 @@ def _parse_linkable_elements(
             entity_links = tuple(EntityReference(element_name=x) for x in structured_name.entity_link_names)
             # Create the spec based on the type of element referenced.
             if TimeDimensionReference(element_name=element_name) in self._known_time_dimension_element_references:
-                if structured_name.time_granularity:
+                if structured_name.time_granularity and not structured_name.date_part:
                     time_dimension_specs.append(
                         TimeDimensionSpec(
                             element_name=element_name,
                             entity_links=entity_links,
                             time_granularity=structured_name.time_granularity,
-                            date_part=structured_name.date_part,
                         )
                     )
+                # If date part is passed, remove requested granularity (to be overridden with default).
                 else:
                     partial_time_dimension_specs.append(
                         PartialTimeDimensionSpec(

metricflow/specs/specs.py

@@ -327,6 +327,7 @@ def qualified_name(self) -> str:  # noqa: D
             entity_link_names=tuple(x.element_name for x in self.entity_links),
             element_name=self.element_name,
             time_granularity=self.time_granularity,
+            date_part=self.date_part,
         ).qualified_name
 
     @property

metricflow/sql/render/big_query.py

@@ -133,11 +133,11 @@ def visit_date_trunc_expr(self, node: SqlDateTruncExpression) -> SqlExpressionRe
     @override
     def render_date_part(self, date_part: DatePart) -> str:
         if date_part == DatePart.DOY:
-            return "DAYOFYEAR"
+            return "dayofyear"
         if date_part == DatePart.DOW:
-            return "DAYOFWEEK"
+            return "dayofweek"
         if date_part == DatePart.WEEK:
-            return "ISOWEEK"
+            return "isoweek"
 
         return super().render_date_part(date_part)
 

metricflow/test/integration/test_cases/itest_metrics.yaml

@@ -1104,3 +1104,16 @@ integration_test:
     ) b
     ON {{ render_date_sub("a", "ds", 5, TimeGranularity.DAY) }} = b.metric_time__day
     GROUP BY metric_time__extract_month
+---
+integration_test:
+  name: date_part_overrides_granularity
+  description: Test query using date_part with incompatible granularity; should override granularity
+  model: SIMPLE_MODEL
+  metrics: ["bookings"]
+  group_by_objs: [{"name": "metric_time", "grain": "month", "date_part": "dow"}]
+  check_query: |
+    SELECT
+      SUM(1) AS bookings
+      , {{ render_extract("ds", DatePart.DOW) }} AS metric_time__extract_dow
+    FROM {{ source_schema }}.fct_bookings
+    GROUP BY {{ render_extract("ds", DatePart.DOW) }};

metricflow/test/integration/test_configured_cases.py

@@ -259,8 +259,11 @@ def test_case(
     for group_by_kwargs in case.group_by_objs:
         kwargs = copy(group_by_kwargs)
         date_part = kwargs.get("date_part")
+        grain = kwargs.get("grain")
         if date_part:
             kwargs["date_part"] = DatePart(date_part)
+        if grain:
+            kwargs["grain"] = TimeGranularity(grain)
         group_by.append(MockQueryParameter(**kwargs))
     query_result = engine.query(
         MetricFlowQueryRequest.create_with_random_request_id(

metricflow/test/plan_conversion/test_dataflow_to_sql_plan.py

@@ -57,6 +57,7 @@
 from metricflow.test.snapshot_utils import assert_plan_snapshot_text_equal
 from metricflow.test.sql.compare_sql_plan import assert_rendered_sql_from_plan_equal, assert_sql_plan_text_equal
 from metricflow.test.time.metric_time_dimension import MTD_SPEC_DAY, MTD_SPEC_QUARTER, MTD_SPEC_WEEK, MTD_SPEC_YEAR
+from metricflow.time.date_part import DatePart
 
 
 @pytest.fixture(scope="session")
@@ -1858,3 +1859,84 @@ def test_derived_offset_cumulative_metric(  # noqa: D
         sql_client=sql_client,
         node=dataflow_plan.sink_output_nodes[0].parent_node,
     )
+
+
+def test_simple_query_with_date_part(  # noqa: D
+    request: FixtureRequest,
+    mf_test_session_state: MetricFlowTestSessionState,
+    dataflow_plan_builder: DataflowPlanBuilder[SemanticModelDataSet],
+    dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter[SemanticModelDataSet],
+    sql_client: SqlClient,
+) -> None:
+    dataflow_plan = dataflow_plan_builder.build_plan(
+        MetricFlowQuerySpec(
+            metric_specs=(MetricSpec(element_name="bookings"),),
+            time_dimension_specs=(
+                DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW),
+            ),
+        )
+    )
+
+    convert_and_check(
+        request=request,
+        mf_test_session_state=mf_test_session_state,
+        dataflow_to_sql_converter=dataflow_to_sql_converter,
+        sql_client=sql_client,
+        node=dataflow_plan.sink_output_nodes[0].parent_node,
+    )
+
+
+def test_simple_query_with_multiple_date_parts(  # noqa: D
+    request: FixtureRequest,
+    mf_test_session_state: MetricFlowTestSessionState,
+    dataflow_plan_builder: DataflowPlanBuilder[SemanticModelDataSet],
+    dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter[SemanticModelDataSet],
+    sql_client: SqlClient,
+) -> None:
+    dataflow_plan = dataflow_plan_builder.build_plan(
+        MetricFlowQuerySpec(
+            metric_specs=(MetricSpec(element_name="bookings"),),
+            time_dimension_specs=(
+                DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DAY),
+                DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW),
+                DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DOY),
+                DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.WEEK),
+                DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.MONTH),
+                DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.QUARTER),
+                DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.YEAR),
+            ),
+        )
+    )
+
+    convert_and_check(
+        request=request,
+        mf_test_session_state=mf_test_session_state,
+        dataflow_to_sql_converter=dataflow_to_sql_converter,
+        sql_client=sql_client,
+        node=dataflow_plan.sink_output_nodes[0].parent_node,
+    )
+
+
+def test_offset_window_with_date_part(  # noqa: D
+    request: FixtureRequest,
+    mf_test_session_state: MetricFlowTestSessionState,
+    dataflow_plan_builder: DataflowPlanBuilder[SemanticModelDataSet],
+    dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter[SemanticModelDataSet],
+    sql_client: SqlClient,
+) -> None:
+    dataflow_plan = dataflow_plan_builder.build_plan(
+        MetricFlowQuerySpec(
+            metric_specs=(MetricSpec(element_name="bookings_growth_2_weeks"),),
+            time_dimension_specs=(
+                DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW),
+            ),
+        )
+    )
+
+    convert_and_check(
+        request=request,
+        mf_test_session_state=mf_test_session_state,
+        dataflow_to_sql_converter=dataflow_to_sql_converter,
+        sql_client=sql_client,
+        node=dataflow_plan.sink_output_nodes[0].parent_node,
+    )

metricflow/test/query/test_query_parser.py

@@ -136,6 +136,18 @@
           - name: revenue
             offset_window: 14 days
             alias: revenue_2_weeks_ago
+    ---
+    metric:
+      name: revenue_since_start_of_year
+      description: Revenue since start of year
+      type: derived
+      type_params:
+        expr: revenue - revenue_start_of_year
+        metrics:
+          - name: revenue
+          - name: revenue
+            offset_to_grain: year
+            alias: revenue_start_of_year
     """
 )
 
@@ -405,22 +417,28 @@ def test_date_part_parsing() -> None:
             group_by=[MockQueryParameter(name="metric_time", date_part=DatePart.DOW)],
         )
 
-    # Date part is incompatible with the requested time granularity for the same time dimension
-    with pytest.raises(RequestTimeGranularityException):
-        query_parser.parse_and_validate_query(
-            metric_names=["revenue"],
-            group_by=[MockQueryParameter(name="metric_time", grain=TimeGranularity.YEAR, date_part=DatePart.MONTH)],
-        )
-
     # Can't query date part for cumulative metrics
     with pytest.raises(UnableToSatisfyQueryError):
         query_parser.parse_and_validate_query(
             metric_names=["revenue_cumulative"],
             group_by=[MockQueryParameter(name="metric_time", date_part=DatePart.DOY)],
         )
 
+    # Can't query date part for metrics with offset to grain
+    with pytest.raises(UnableToSatisfyQueryError):
+        query_parser.parse_and_validate_query(
+            metric_names=["revenue_since_start_of_year"],
+            group_by=[MockQueryParameter(name="metric_time", date_part=DatePart.DAY)],
+        )
+
     # Date part is compatible
     query_parser.parse_and_validate_query(
         metric_names=["revenue"],
         group_by=[MockQueryParameter(name="metric_time", date_part=DatePart.MONTH)],
     )
+
+    # Incompatible granularity gets overriden
+    query_parser.parse_and_validate_query(
+        metric_names=["revenue"],
+        group_by=[MockQueryParameter(name="metric_time", grain=TimeGranularity.YEAR, date_part=DatePart.MONTH)],
+    )

metricflow/test/snapshots/test_convert_semantic_model.py/SqlQueryPlan/BigQuery/test_convert_query_semantic_model__plan0.sql

@@ -9,10 +9,10 @@ SELECT
   , EXTRACT(year FROM revenue_src_10006.created_at) AS ds__extract_year
   , EXTRACT(quarter FROM revenue_src_10006.created_at) AS ds__extract_quarter
   , EXTRACT(month FROM revenue_src_10006.created_at) AS ds__extract_month
-  , EXTRACT(ISOWEEK FROM revenue_src_10006.created_at) AS ds__extract_week
+  , EXTRACT(isoweek FROM revenue_src_10006.created_at) AS ds__extract_week
   , EXTRACT(day FROM revenue_src_10006.created_at) AS ds__extract_day
-  , EXTRACT(DAYOFWEEK FROM revenue_src_10006.created_at) AS ds__extract_dow
-  , EXTRACT(DAYOFYEAR FROM revenue_src_10006.created_at) AS ds__extract_doy
+  , EXTRACT(dayofweek FROM revenue_src_10006.created_at) AS ds__extract_dow
+  , EXTRACT(dayofyear FROM revenue_src_10006.created_at) AS ds__extract_doy
   , revenue_src_10006.created_at AS company__ds__day
   , DATE_TRUNC(revenue_src_10006.created_at, isoweek) AS company__ds__week
   , DATE_TRUNC(revenue_src_10006.created_at, month) AS company__ds__month
@@ -21,10 +21,10 @@ SELECT
   , EXTRACT(year FROM revenue_src_10006.created_at) AS company__ds__extract_year
   , EXTRACT(quarter FROM revenue_src_10006.created_at) AS company__ds__extract_quarter
   , EXTRACT(month FROM revenue_src_10006.created_at) AS company__ds__extract_month
-  , EXTRACT(ISOWEEK FROM revenue_src_10006.created_at) AS company__ds__extract_week
+  , EXTRACT(isoweek FROM revenue_src_10006.created_at) AS company__ds__extract_week
   , EXTRACT(day FROM revenue_src_10006.created_at) AS company__ds__extract_day
-  , EXTRACT(DAYOFWEEK FROM revenue_src_10006.created_at) AS company__ds__extract_dow
-  , EXTRACT(DAYOFYEAR FROM revenue_src_10006.created_at) AS company__ds__extract_doy
+  , EXTRACT(dayofweek FROM revenue_src_10006.created_at) AS company__ds__extract_dow
+  , EXTRACT(dayofyear FROM revenue_src_10006.created_at) AS company__ds__extract_doy
   , revenue_src_10006.user_id AS user
   , revenue_src_10006.user_id AS company__user
 FROM ***************************.fct_revenue revenue_src_10006

metricflow/test/snapshots/test_convert_semantic_model.py/SqlQueryPlan/BigQuery/test_convert_table_semantic_model_with_measures__plan0.sql

@@ -9,10 +9,10 @@ SELECT
   , EXTRACT(year FROM id_verifications_src_10003.ds) AS ds__extract_year
   , EXTRACT(quarter FROM id_verifications_src_10003.ds) AS ds__extract_quarter
   , EXTRACT(month FROM id_verifications_src_10003.ds) AS ds__extract_month
-  , EXTRACT(ISOWEEK FROM id_verifications_src_10003.ds) AS ds__extract_week
+  , EXTRACT(isoweek FROM id_verifications_src_10003.ds) AS ds__extract_week
   , EXTRACT(day FROM id_verifications_src_10003.ds) AS ds__extract_day
-  , EXTRACT(DAYOFWEEK FROM id_verifications_src_10003.ds) AS ds__extract_dow
-  , EXTRACT(DAYOFYEAR FROM id_verifications_src_10003.ds) AS ds__extract_doy
+  , EXTRACT(dayofweek FROM id_verifications_src_10003.ds) AS ds__extract_dow
+  , EXTRACT(dayofyear FROM id_verifications_src_10003.ds) AS ds__extract_doy
   , id_verifications_src_10003.ds_partitioned AS ds_partitioned__day
   , DATE_TRUNC(id_verifications_src_10003.ds_partitioned, isoweek) AS ds_partitioned__week
   , DATE_TRUNC(id_verifications_src_10003.ds_partitioned, month) AS ds_partitioned__month
@@ -21,10 +21,10 @@ SELECT
   , EXTRACT(year FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_year
   , EXTRACT(quarter FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_quarter
   , EXTRACT(month FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_month
-  , EXTRACT(ISOWEEK FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_week
+  , EXTRACT(isoweek FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_week
   , EXTRACT(day FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_day
-  , EXTRACT(DAYOFWEEK FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_dow
-  , EXTRACT(DAYOFYEAR FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_doy
+  , EXTRACT(dayofweek FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_dow
+  , EXTRACT(dayofyear FROM id_verifications_src_10003.ds_partitioned) AS ds_partitioned__extract_doy
   , id_verifications_src_10003.verification_type
   , id_verifications_src_10003.ds AS verification__ds__day
   , DATE_TRUNC(id_verifications_src_10003.ds, isoweek) AS verification__ds__week
@@ -34,10 +34,10 @@ SELECT
   , EXTRACT(year FROM id_verifications_src_10003.ds) AS verification__ds__extract_year
   , EXTRACT(quarter FROM id_verifications_src_10003.ds) AS verification__ds__extract_quarter
   , EXTRACT(month FROM id_verifications_src_10003.ds) AS verification__ds__extract_month
-  , EXTRACT(ISOWEEK FROM id_verifications_src_10003.ds) AS verification__ds__extract_week
+  , EXTRACT(isoweek FROM id_verifications_src_10003.ds) AS verification__ds__extract_week
   , EXTRACT(day FROM id_verifications_src_10003.ds) AS verification__ds__extract_day
-  , EXTRACT(DAYOFWEEK FROM id_verifications_src_10003.ds) AS verification__ds__extract_dow
-  , EXTRACT(DAYOFYEAR FROM id_verifications_src_10003.ds) AS verification__ds__extract_doy
+  , EXTRACT(dayofweek FROM id_verifications_src_10003.ds) AS verification__ds__extract_dow
+  , EXTRACT(dayofyear FROM id_verifications_src_10003.ds) AS verification__ds__extract_doy
   , id_verifications_src_10003.ds_partitioned AS verification__ds_partitioned__day
   , DATE_TRUNC(id_verifications_src_10003.ds_partitioned, isoweek) AS verification__ds_partitioned__week
   , DATE_TRUNC(id_verifications_src_10003.ds_partitioned, month) AS verification__ds_partitioned__month
@@ -46,10 +46,10 @@ SELECT
   , EXTRACT(year FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_year
   , EXTRACT(quarter FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_quarter
   , EXTRACT(month FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_month
-  , EXTRACT(ISOWEEK FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_week
+  , EXTRACT(isoweek FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_week
   , EXTRACT(day FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_day
-  , EXTRACT(DAYOFWEEK FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_dow
-  , EXTRACT(DAYOFYEAR FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_doy
+  , EXTRACT(dayofweek FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_dow
+  , EXTRACT(dayofyear FROM id_verifications_src_10003.ds_partitioned) AS verification__ds_partitioned__extract_doy
   , id_verifications_src_10003.verification_type AS verification__verification_type
   , id_verifications_src_10003.verification_id AS verification
   , id_verifications_src_10003.user_id AS user