Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split up test_dataflow_to_sql_plan module into more manageable components #836

Merged
merged 4 commits into from
Nov 3, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions metricflow/test/fixtures/sql_fixtures.py
Original file line number Diff line number Diff line change
@@ -2,9 +2,22 @@

import pytest

from metricflow.model.semantic_manifest_lookup import SemanticManifestLookup
from metricflow.plan_conversion.column_resolver import DunderColumnAssociationResolver
from metricflow.plan_conversion.dataflow_to_sql import DataflowToSqlQueryPlanConverter
from metricflow.sql.render.sql_plan_renderer import DefaultSqlQueryPlanRenderer, SqlQueryPlanRenderer


@pytest.fixture
def default_sql_plan_renderer() -> SqlQueryPlanRenderer: # noqa: D
return DefaultSqlQueryPlanRenderer()


@pytest.fixture(scope="session")
def dataflow_to_sql_converter( # noqa: D
simple_semantic_manifest_lookup: SemanticManifestLookup,
) -> DataflowToSqlQueryPlanConverter:
return DataflowToSqlQueryPlanConverter(
column_association_resolver=DunderColumnAssociationResolver(simple_semantic_manifest_lookup),
semantic_manifest_lookup=simple_semantic_manifest_lookup,
)
4 changes: 2 additions & 2 deletions metricflow/test/generate_snapshots.py
Original file line number Diff line number Diff line change
@@ -121,8 +121,8 @@ def run_tests(test_configuration: MetricFlowTestConfiguration) -> None: # noqa:
os.environ["MF_SQL_ENGINE_PASSWORD"] = test_configuration.credential_set.engine_password

if test_configuration.engine is SqlEngine.DUCKDB:
# Can't use --use-persistent-source-schema with duckdb since it's in memory.
run_command(f"pytest -x -vv -n 4 --overwrite-snapshots -m '{SQL_ENGINE_SNAPSHOT_MARKER_NAME}' {TEST_DIRECTORY}")
# DuckDB is fast, so generate all snapshots, including the engine-agnostic ones
run_command(f"pytest -x -vv -n 4 --overwrite-snapshots -k 'not itest' {TEST_DIRECTORY}")
elif (
test_configuration.engine is SqlEngine.REDSHIFT
or test_configuration.engine is SqlEngine.SNOWFLAKE
17 changes: 0 additions & 17 deletions metricflow/test/plan_conversion/conftest.py

This file was deleted.

686 changes: 109 additions & 577 deletions metricflow/test/plan_conversion/test_dataflow_to_sql_plan.py

Large diffs are not rendered by default.

Empty file.
66 changes: 66 additions & 0 deletions metricflow/test/query_rendering/compare_rendered_query.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
"""Centralizes useful functions for comparing two different rendered queries."""
from __future__ import annotations

from _pytest.fixtures import FixtureRequest

from metricflow.dataflow.dataflow_plan import BaseOutput
from metricflow.plan_conversion.dataflow_to_sql import DataflowToSqlQueryPlanConverter
from metricflow.protocols.sql_client import SqlClient
from metricflow.sql.optimizer.optimization_levels import SqlQueryOptimizationLevel
from metricflow.test.dataflow_plan_to_svg import display_graph_if_requested
from metricflow.test.fixtures.setup_fixtures import MetricFlowTestSessionState
from metricflow.test.sql.compare_sql_plan import assert_rendered_sql_from_plan_equal


def convert_and_check(
request: FixtureRequest,
mf_test_session_state: MetricFlowTestSessionState,
dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter,
sql_client: SqlClient,
node: BaseOutput,
) -> None:
"""Renders an engine-specific query output from a BaseOutput DataFlowPlan node.
TODO: refine interface once file move operations are complete.
"""
# Run dataflow -> sql conversion without optimizers
sql_query_plan = dataflow_to_sql_converter.convert_to_sql_query_plan(
sql_engine_type=sql_client.sql_engine_type,
sql_query_plan_id="plan0",
dataflow_plan_node=node,
optimization_level=SqlQueryOptimizationLevel.O0,
)

display_graph_if_requested(
request=request,
mf_test_session_state=mf_test_session_state,
dag_graph=sql_query_plan,
)

assert_rendered_sql_from_plan_equal(
request=request,
mf_test_session_state=mf_test_session_state,
sql_query_plan=sql_query_plan,
sql_client=sql_client,
)

# Run dataflow -> sql conversion with optimizers
sql_query_plan = dataflow_to_sql_converter.convert_to_sql_query_plan(
sql_engine_type=sql_client.sql_engine_type,
sql_query_plan_id="plan0_optimized",
dataflow_plan_node=node,
optimization_level=SqlQueryOptimizationLevel.O4,
)

display_graph_if_requested(
request=request,
mf_test_session_state=mf_test_session_state,
dag_graph=sql_query_plan,
)

assert_rendered_sql_from_plan_equal(
request=request,
mf_test_session_state=mf_test_session_state,
sql_query_plan=sql_query_plan,
sql_client=sql_client,
)
502 changes: 502 additions & 0 deletions metricflow/test/query_rendering/test_query_rendering.py

Large diffs are not rendered by default.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.