Update snapshots & test SQL

dbt-labs · Nov 3, 2023 · 7aa4682 · 7aa4682
1 parent fab0350
commit 7aa4682
Show file tree

Hide file tree

Showing 31 changed files with 188 additions and 356 deletions.
diff --git a/metricflow/dataflow/builder/dataflow_plan_builder.py b/metricflow/dataflow/builder/dataflow_plan_builder.py
@@ -241,14 +241,14 @@ def _build_metrics_output_node(
                     f"For {metric.type} metric: {metric_spec}, needed metrics are:\n"
                     f"{pformat_big_objects(metric_input_specs=metric_input_specs)}"
                 )
-
+                join_type = SqlJoinType.FULL_OUTER if metric.type is MetricType.DERIVED else SqlJoinType.INNER
                 compute_metrics_node = ComputeMetricsNode(
                     parent_node=self._build_metrics_output_node(
                         metric_specs=metric_input_specs,
                         queried_linkable_specs=queried_linkable_specs,
                         where_constraint=where_constraint,
                         time_range_constraint=time_range_constraint,
-                        combine_metrics_join_type=SqlJoinType.FULL_OUTER if MetricType.DERIVED else SqlJoinType.INNER,
+                        combine_metrics_join_type=join_type,
                     ),
                     metric_specs=[metric_spec],
                 )

diff --git a/metricflow/test/integration/test_cases/itest_metrics.yaml b/metricflow/test/integration/test_cases/itest_metrics.yaml
@@ -42,7 +42,7 @@ integration_test:
       FROM {{source_schema}}.fct_bookings
       GROUP BY ds
     ) b
-    JOIN (
+    FULL OUTER JOIN (
       SELECT
         SUM(1) AS views
         , ds
@@ -519,7 +519,7 @@ integration_test:
       GROUP BY
         ds
     ) a
-    INNER JOIN (
+    FULL OUTER JOIN (
       SELECT
         SUM(1) AS lux_listings
         , created_at AS metric_time__day
@@ -654,7 +654,7 @@ integration_test:
       ON a.listing_id = b.listing_id
       GROUP BY 2
     ) bk
-    INNER JOIN (
+    FULL OUTER JOIN (
       SELECT
         SUM(1) AS views
         ,d.is_lux
@@ -683,7 +683,7 @@ integration_test:
       GROUP BY
         metric_time__day
     ) a
-    INNER JOIN (
+    FULL OUTER JOIN (
       SELECT
         c.ds AS metric_time__day
         , d.bookings_2_weeks_ago AS bookings_2_weeks_ago
@@ -718,7 +718,7 @@ integration_test:
       GROUP BY
         metric_time__day
     ) a
-    INNER JOIN (
+    FULL OUTER JOIN (
       SELECT
         c.ds AS metric_time__day
         , d.bookings_at_start_of_month AS bookings_at_start_of_month
@@ -760,7 +760,7 @@ integration_test:
       ) f
       ON {{ render_date_sub("g", "ds", 1, TimeGranularity.MONTH) }} = f.metric_time__day
     ) a
-    INNER JOIN (
+    FULL OUTER JOIN (
       SELECT
         c.ds AS metric_time__day
         , d.bookings AS month_start_bookings
@@ -808,7 +808,7 @@ integration_test:
   check_query: |
     SELECT
       booking_value - instant_booking_value AS booking_value_sub_instant
-      , a.metric_time__day
+      , COALESCE(a.metric_time__day, b.metric_time__day) AS metric_time__day
     FROM (
       SELECT
         SUM(booking_value) AS instant_booking_value
@@ -818,7 +818,7 @@ integration_test:
       GROUP BY
         ds
     ) a
-    INNER JOIN (
+    FULL OUTER JOIN (
       SELECT
         SUM(booking_value) AS booking_value
         , ds AS metric_time__day
@@ -843,7 +843,7 @@ integration_test:
     FROM (
       SELECT
         booking_value - instant_booking_value AS booking_value_sub_instant
-        , a.metric_time__day AS metric_time__day
+        , COALESCE(a.metric_time__day, b.metric_time__day) AS metric_time__day
       FROM (
         SELECT
           SUM(booking_value) AS instant_booking_value
@@ -853,7 +853,7 @@ integration_test:
         GROUP BY
           ds
       ) a
-      INNER JOIN (
+      FULL OUTER JOIN (
         SELECT
           SUM(booking_value) AS booking_value
           , ds AS metric_time__day
@@ -901,7 +901,7 @@ integration_test:
       FROM {{ source_schema }}.fct_bookings
       GROUP BY metric_time__week
     ) a
-    INNER JOIN (
+    FULL OUTER JOIN (
       SELECT
         {{ render_date_trunc("c.ds", TimeGranularity.WEEK) }} AS metric_time__week
         , SUM(d.bookings_at_start_of_month) AS bookings_at_start_of_month
@@ -944,7 +944,7 @@ integration_test:
       ON {{ render_date_sub("g", "ds", 1, TimeGranularity.MONTH) }} = f.metric_time__day
       GROUP BY metric_time__year
     ) a
-    INNER JOIN (
+    FULL OUTER JOIN (
       SELECT
         {{ render_date_trunc("c.ds", TimeGranularity.YEAR) }} AS metric_time__year
         , SUM(d.bookings) AS month_start_bookings
@@ -1024,7 +1024,7 @@ integration_test:
       FROM {{ source_schema }}.fct_bookings
       GROUP BY metric_time__week, metric_time__month
     ) a
-    INNER JOIN (
+    FULL OUTER JOIN (
       SELECT
         {{ render_date_trunc("c.ds", TimeGranularity.WEEK) }} AS metric_time__week
         , {{ render_date_trunc("c.ds", TimeGranularity.MONTH) }} AS metric_time__month
@@ -1290,7 +1290,7 @@ integration_test:
         ) subq_3
         ON subq_5.ds = subq_3.metric_time__day
       ) subq_7
-      INNER JOIN (
+      FULL OUTER JOIN (
         SELECT
           subq_11.ds AS metric_time__day
           , SUM(subq_9.bookings) AS bookings_2_weeks_ago

diff --git a/...test_dataflow_plan_builder.py/DataflowPlan/test_derived_metric_offset_to_grain__dfp_0.xml b/...test_dataflow_plan_builder.py/DataflowPlan/test_derived_metric_offset_to_grain__dfp_0.xml
@@ -15,7 +15,8 @@
             <CombineMetricsNode>
                 <!-- description = Combine Metrics -->
                 <!-- node_id = cbm_0 -->
-                <!-- join type = SqlJoinType.INNER -->
+                <!-- join type = SqlJoinType.FULL_OUTER -->
+                <!-- de-duplication method = post-join aggregation across all dimensions -->
                 <ComputeMetricsNode>
                     <!-- description = Compute Metrics via Expressions -->
                     <!-- node_id = cm_0 -->

diff --git a/...t_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_derived_metric__dfp_0.xml b/...t_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_derived_metric__dfp_0.xml
@@ -15,7 +15,8 @@
             <CombineMetricsNode>
                 <!-- description = Combine Metrics -->
                 <!-- node_id = cbm_0 -->
-                <!-- join type = SqlJoinType.INNER -->
+                <!-- join type = SqlJoinType.FULL_OUTER -->
+                <!-- de-duplication method = post-join aggregation across all dimensions -->
                 <ComputeMetricsNode>
                     <!-- description = Compute Metrics via Expressions -->
                     <!-- node_id = cm_0 -->

diff --git a/...pshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml b/...pshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml
@@ -15,7 +15,8 @@
             <CombineMetricsNode>
                 <!-- description = Combine Metrics -->
                 <!-- node_id = cbm_0 -->
-                <!-- join type = SqlJoinType.INNER -->
+                <!-- join type = SqlJoinType.FULL_OUTER -->
+                <!-- de-duplication method = post-join aggregation across all dimensions -->
                 <ComputeMetricsNode>
                     <!-- description = Compute Metrics via Expressions -->
                     <!-- node_id = cm_0 -->

diff --git a/...shots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric__plan0.sql b/...shots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric__plan0.sql
@@ -6,8 +6,8 @@ FROM (
   -- Combine Metrics
   SELECT
     COALESCE(subq_4.metric_time__day, subq_9.metric_time__day) AS metric_time__day
-    , subq_4.ref_bookings AS ref_bookings
-    , subq_9.bookings AS bookings
+    , MAX(subq_4.ref_bookings) AS ref_bookings
+    , MAX(subq_9.bookings) AS bookings
   FROM (
     -- Compute Metrics via Expressions
     SELECT
@@ -224,7 +224,7 @@ FROM (
         subq_2.metric_time__day
     ) subq_3
   ) subq_4
-  INNER JOIN (
+  FULL OUTER JOIN (
     -- Compute Metrics via Expressions
     SELECT
       subq_8.metric_time__day
@@ -441,13 +441,7 @@ FROM (
     ) subq_8
   ) subq_9
   ON
-    (
-      subq_4.metric_time__day = subq_9.metric_time__day
-    ) OR (
-      (
-        subq_4.metric_time__day IS NULL
-      ) AND (
-        subq_9.metric_time__day IS NULL
-      )
-    )
+    subq_4.metric_time__day = subq_9.metric_time__day
+  GROUP BY
+    COALESCE(subq_4.metric_time__day, subq_9.metric_time__day)
 ) subq_10
diff --git a/..._derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric__plan0_optimized.sql b/..._derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric__plan0_optimized.sql
@@ -6,8 +6,8 @@ FROM (
   -- Combine Metrics
   SELECT
     COALESCE(subq_15.metric_time__day, subq_20.metric_time__day) AS metric_time__day
-    , subq_15.ref_bookings AS ref_bookings
-    , subq_20.bookings AS bookings
+    , MAX(subq_15.ref_bookings) AS ref_bookings
+    , MAX(subq_20.bookings) AS bookings
   FROM (
     -- Aggregate Measures
     -- Compute Metrics via Expressions
@@ -27,7 +27,7 @@ FROM (
     GROUP BY
       metric_time__day
   ) subq_15
-  INNER JOIN (
+  FULL OUTER JOIN (
     -- Aggregate Measures
     -- Compute Metrics via Expressions
     SELECT
@@ -47,13 +47,7 @@ FROM (
       metric_time__day
   ) subq_20
   ON
-    (
-      subq_15.metric_time__day = subq_20.metric_time__day
-    ) OR (
-      (
-        subq_15.metric_time__day IS NULL
-      ) AND (
-        subq_20.metric_time__day IS NULL
-      )
-    )
+    subq_15.metric_time__day = subq_20.metric_time__day
+  GROUP BY
+    COALESCE(subq_15.metric_time__day, subq_20.metric_time__day)
 ) subq_21
diff --git a/...tric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain__plan0.sql b/...tric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain__plan0.sql
@@ -6,8 +6,8 @@ FROM (
   -- Combine Metrics
   SELECT
     COALESCE(subq_4.metric_time__day, subq_12.metric_time__day) AS metric_time__day
-    , subq_4.bookings AS bookings
-    , subq_12.bookings_at_start_of_month AS bookings_at_start_of_month
+    , MAX(subq_4.bookings) AS bookings
+    , MAX(subq_12.bookings_at_start_of_month) AS bookings_at_start_of_month
   FROM (
     -- Compute Metrics via Expressions
     SELECT
@@ -224,7 +224,7 @@ FROM (
         subq_2.metric_time__day
     ) subq_3
   ) subq_4
-  INNER JOIN (
+  FULL OUTER JOIN (
     -- Compute Metrics via Expressions
     SELECT
       subq_11.metric_time__day
@@ -541,13 +541,7 @@ FROM (
     ) subq_11
   ) subq_12
   ON
-    (
-      subq_4.metric_time__day = subq_12.metric_time__day
-    ) OR (
-      (
-        subq_4.metric_time__day IS NULL
-      ) AND (
-        subq_12.metric_time__day IS NULL
-      )
-    )
+    subq_4.metric_time__day = subq_12.metric_time__day
+  GROUP BY
+    COALESCE(subq_4.metric_time__day, subq_12.metric_time__day)
 ) subq_13
diff --git a/...ring.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain__plan0_optimized.sql b/...ring.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain__plan0_optimized.sql
@@ -6,8 +6,8 @@ FROM (
   -- Combine Metrics
   SELECT
     COALESCE(subq_18.metric_time__day, subq_26.metric_time__day) AS metric_time__day
-    , subq_18.bookings AS bookings
-    , subq_26.bookings_at_start_of_month AS bookings_at_start_of_month
+    , MAX(subq_18.bookings) AS bookings
+    , MAX(subq_26.bookings_at_start_of_month) AS bookings_at_start_of_month
   FROM (
     -- Aggregate Measures
     -- Compute Metrics via Expressions
@@ -27,7 +27,7 @@ FROM (
     GROUP BY
       metric_time__day
   ) subq_18
-  INNER JOIN (
+  FULL OUTER JOIN (
     -- Join to Time Spine Dataset
     -- Pass Only Elements:
     --   ['bookings', 'metric_time__day']
@@ -51,13 +51,7 @@ FROM (
       subq_22.ds
   ) subq_26
   ON
-    (
-      subq_18.metric_time__day = subq_26.metric_time__day
-    ) OR (
-      (
-        subq_18.metric_time__day IS NULL
-      ) AND (
-        subq_26.metric_time__day IS NULL
-      )
-    )
+    subq_18.metric_time__day = subq_26.metric_time__day
+  GROUP BY
+    COALESCE(subq_18.metric_time__day, subq_26.metric_time__day)
 ) subq_27
diff --git a/...y/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain_and_granularity__plan0.sql b/...y/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain_and_granularity__plan0.sql
@@ -6,8 +6,8 @@ FROM (
   -- Combine Metrics
   SELECT
     COALESCE(subq_4.metric_time__week, subq_12.metric_time__week) AS metric_time__week
-    , subq_4.bookings AS bookings
-    , subq_12.bookings_at_start_of_month AS bookings_at_start_of_month
+    , MAX(subq_4.bookings) AS bookings
+    , MAX(subq_12.bookings_at_start_of_month) AS bookings_at_start_of_month
   FROM (
     -- Compute Metrics via Expressions
     SELECT
@@ -224,7 +224,7 @@ FROM (
         subq_2.metric_time__week
     ) subq_3
   ) subq_4
-  INNER JOIN (
+  FULL OUTER JOIN (
     -- Compute Metrics via Expressions
     SELECT
       subq_11.metric_time__week
@@ -542,13 +542,7 @@ FROM (
     ) subq_11
   ) subq_12
   ON
-    (
-      subq_4.metric_time__week = subq_12.metric_time__week
-    ) OR (
-      (
-        subq_4.metric_time__week IS NULL
-      ) AND (
-        subq_12.metric_time__week IS NULL
-      )
-    )
+    subq_4.metric_time__week = subq_12.metric_time__week
+  GROUP BY
+    COALESCE(subq_4.metric_time__week, subq_12.metric_time__week)
 ) subq_13
diff --git a/...Plan/DuckDB/test_derived_metric_with_offset_to_grain_and_granularity__plan0_optimized.sql b/...Plan/DuckDB/test_derived_metric_with_offset_to_grain_and_granularity__plan0_optimized.sql
@@ -6,8 +6,8 @@ FROM (
   -- Combine Metrics
   SELECT
     COALESCE(subq_18.metric_time__week, subq_26.metric_time__week) AS metric_time__week
-    , subq_18.bookings AS bookings
-    , subq_26.bookings_at_start_of_month AS bookings_at_start_of_month
+    , MAX(subq_18.bookings) AS bookings
+    , MAX(subq_26.bookings_at_start_of_month) AS bookings_at_start_of_month
   FROM (
     -- Aggregate Measures
     -- Compute Metrics via Expressions
@@ -27,7 +27,7 @@ FROM (
     GROUP BY
       metric_time__week
   ) subq_18
-  INNER JOIN (
+  FULL OUTER JOIN (
     -- Join to Time Spine Dataset
     -- Pass Only Elements:
     --   ['bookings', 'metric_time__week']
@@ -52,13 +52,7 @@ FROM (
       DATE_TRUNC('week', subq_22.ds)
   ) subq_26
   ON
-    (
-      subq_18.metric_time__week = subq_26.metric_time__week
-    ) OR (
-      (
-        subq_18.metric_time__week IS NULL
-      ) AND (
-        subq_26.metric_time__week IS NULL
-      )
-    )
+    subq_18.metric_time__week = subq_26.metric_time__week
+  GROUP BY
+    COALESCE(subq_18.metric_time__week, subq_26.metric_time__week)
 ) subq_27