Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dedupe filter specs in join to time spine queries #1578

Open
wants to merge 1 commit into
base: court/custom-offset3
Choose a base branch
from
+197 −228
Open

Dedupe filter specs in join to time spine queries #1578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion metricflow/dataflow/builder/dataflow_plan_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1648,7 +1648,9 @@ def _build_aggregated_measure_from_measure_source_node(
join_on_time_dimension_spec = self._determine_time_spine_join_spec(
measure_properties=measure_properties, required_time_spine_specs=base_queried_agg_time_dimension_specs
)
required_time_spine_specs = (join_on_time_dimension_spec,) + base_queried_agg_time_dimension_specs
required_time_spine_specs = base_queried_agg_time_dimension_specs
if join_on_time_dimension_spec not in required_time_spine_specs:
required_time_spine_specs = (join_on_time_dimension_spec,) + required_time_spine_specs
time_spine_node = self._build_time_spine_node(required_time_spine_specs)
unaggregated_measure_node = JoinToTimeSpineNode.create(
metric_source_node=unaggregated_measure_node,
Expand Down
2 changes: 1 addition & 1 deletion ...tom_granularity.py/SqlPlan/BigQuery/test_offset_metric_with_custom_granularity__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...lan/BigQuery/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...m_granularity.py/SqlPlan/Databricks/test_offset_metric_with_custom_granularity__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...n/Databricks/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...ustom_granularity.py/SqlPlan/DuckDB/test_offset_metric_with_custom_granularity__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...lPlan/DuckDB/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...tom_granularity.py/SqlPlan/Postgres/test_offset_metric_with_custom_granularity__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...lan/Postgres/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...tom_granularity.py/SqlPlan/Redshift/test_offset_metric_with_custom_granularity__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...lan/Redshift/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...om_granularity.py/SqlPlan/Snowflake/test_offset_metric_with_custom_granularity__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...an/Snowflake/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...custom_granularity.py/SqlPlan/Trino/test_offset_metric_with_custom_granularity__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...qlPlan/Trino/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
7 changes: 1 addition & 6 deletions ...test_dataflow_plan_builder.py/DataflowPlan/test_derived_metric_offset_to_grain__dfp_0.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,18 +96,13 @@ docstring:
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
<FilterElementsNode>
<!-- description = "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_1') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
7 changes: 1 addition & 6 deletions ...s/test_dataflow_plan_builder.py/DataflowPlan/test_derived_metric_offset_window__dfp_0.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,18 +62,13 @@ docstring:
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
<FilterElementsNode>
<!-- description = "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_0') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
7 changes: 1 addition & 6 deletions ...st_dataflow_plan_builder.py/DataflowPlan/test_derived_offset_cumulative_metric__dfp_0.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,18 +73,13 @@ test_filename: test_dataflow_plan_builder.py
</MetricTimeDimensionTransformNode>
</JoinOverTimeRangeNode>
<FilterElementsNode>
<!-- description = "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_0') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
11 changes: 1 addition & 10 deletions ...t_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_derived_metric__dfp_0.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,7 @@ test_filename: test_dataflow_plan_builder.py
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
<FilterElementsNode>
<!-- description = -->
<!-- "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_2') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
Expand All @@ -180,14 +179,6 @@ test_filename: test_dataflow_plan_builder.py
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity( -->
<!-- name='day', -->
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
11 changes: 1 addition & 10 deletions ...flow_plan_builder.py/DataflowPlan/test_nested_derived_metric_with_outer_offset__dfp_0.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,7 @@ test_filename: test_dataflow_plan_builder.py
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
<FilterElementsNode>
<!-- description = -->
<!-- "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_0') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
Expand All @@ -96,14 +95,6 @@ test_filename: test_dataflow_plan_builder.py
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity( -->
<!-- name='day', -->
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
2 changes: 1 addition & 1 deletion ...ng.py/SqlPlan/BigQuery/test_cumulative_time_offset_metric_with_time_constraint__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ FROM (
, subq_4.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_4.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_6.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...py/SqlPlan/BigQuery/test_derived_metric_with_month_dimension_and_offset_window__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ FROM (
, subq_1.booking_monthly__listing AS booking_monthly__listing
, subq_1.bookings_monthly AS bookings_monthly
FROM (
-- Pass Only Elements: ['metric_time__month', 'metric_time__month']
-- Pass Only Elements: ['metric_time__month',]
SELECT
subq_3.metric_time__month
FROM (
Expand Down
2 changes: 1 addition & 1 deletion .../BigQuery/test_derived_metric_with_month_dimension_and_offset_window__plan0_optimized.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
-- Change Column Aliases
-- Pass Only Elements: ['metric_time__month', 'metric_time__month']
-- Pass Only Elements: ['metric_time__month',]
SELECT
DATETIME_TRUNC(ds, month) AS metric_time__month
FROM ***************************.mf_time_spine time_spine_src_16006
Expand Down
2 changes: 1 addition & 1 deletion ..._metric_rendering.py/SqlPlan/BigQuery/test_derived_metric_with_offset_to_grain__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ FROM (
, subq_6.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_6.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_8.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...ed_metric_rendering.py/SqlPlan/BigQuery/test_derived_metric_with_offset_window__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ FROM (
, subq_6.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_6.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_8.metric_time__day
FROM (
Expand Down
4 changes: 2 additions & 2 deletions ...py/SqlPlan/BigQuery/test_derived_metric_with_offset_window_and_offset_to_grain__plan0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ FROM (
, subq_1.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down Expand Up @@ -486,7 +486,7 @@ FROM (
, subq_10.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_10.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_12.metric_time__day
FROM (
Expand Down
Loading
Loading