Add test cases for saved queries in the CLI.

metricflow/test/cli/test_cli.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
+import logging
 import shutil
 import textwrap
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Iterator
 
 import pytest
+from _pytest.fixtures import FixtureRequest
 from dbt_semantic_interfaces.parsing.dir_to_model import (
     parse_yaml_files_to_validation_ready_semantic_manifest,
 )
@@ -23,9 +25,16 @@
     tutorial,
     validate_configs,
 )
-from metricflow.protocols.sql_client import SqlEngine
+from metricflow.protocols.sql_client import SqlClient, SqlEngine
 from metricflow.test.fixtures.cli_fixtures import MetricFlowCliRunner
+from metricflow.test.fixtures.setup_fixtures import MetricFlowTestSessionState
 from metricflow.test.model.example_project_configuration import EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE
+from metricflow.test.snapshot_utils import assert_object_snapshot_equal
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: Use snapshots to compare CLI output for all tests here.
 
 
 def test_query(capsys: pytest.CaptureFixture, cli_runner: MetricFlowCliRunner) -> None:  # noqa: D
@@ -144,3 +153,107 @@ def test_list_entities(capsys: pytest.CaptureFixture, cli_runner: MetricFlowCliR
 
     assert "guest" in resp.output
     assert "host" in resp.output
+
+
+def test_saved_query(  # noqa: D
+    request: FixtureRequest,
+    capsys: pytest.CaptureFixture,
+    mf_test_session_state: MetricFlowTestSessionState,
+    cli_runner: MetricFlowCliRunner,
+    sql_client: SqlClient,
+) -> None:
+    # Disabling capsys to resolve error "ValueError: I/O operation on closed file". Better solution TBD.
+    with capsys.disabled():
+        resp = cli_runner.run(
+            query, args=["--saved-query", "p0_booking", "--order", "metric_time__day,listing__capacity_latest"]
+        )
+
+    assert resp.exit_code == 0
+
+    assert_object_snapshot_equal(
+        request=request,
+        mf_test_session_state=mf_test_session_state,
+        obj_id="cli_output",
+        obj=resp.output,
+        sql_client=sql_client,
+    )
+
+
+def test_saved_query_with_where(  # noqa: D
+    request: FixtureRequest,
+    capsys: pytest.CaptureFixture,
+    mf_test_session_state: MetricFlowTestSessionState,
+    cli_runner: MetricFlowCliRunner,
+    sql_client: SqlClient,
+) -> None:
+    # Disabling capsys to resolve error "ValueError: I/O operation on closed file". Better solution TBD.
+    with capsys.disabled():
+        resp = cli_runner.run(
+            query,
+            args=[
+                "--saved-query",
+                "p0_booking",
+                "--order",
+                "metric_time__day,listing__capacity_latest",
+                "--where",
+                "{{ Dimension('listing__capacity_latest') }} > 4",
+            ],
+        )
+
+    assert resp.exit_code == 0
+
+    assert_object_snapshot_equal(
+        request=request,
+        mf_test_session_state=mf_test_session_state,
+        obj_id="cli_output",
+        obj=resp.output,
+        sql_client=sql_client,
+    )
+
+
+def test_saved_query_with_limit(  # noqa: D
+    request: FixtureRequest,
+    capsys: pytest.CaptureFixture,
+    mf_test_session_state: MetricFlowTestSessionState,
+    cli_runner: MetricFlowCliRunner,
+    sql_client: SqlClient,
+) -> None:
+    # Disabling capsys to resolve error "ValueError: I/O operation on closed file". Better solution TBD.
+    with capsys.disabled():
+        resp = cli_runner.run(
+            query,
+            args=[
+                "--saved-query",
+                "p0_booking",
+                "--order",
+                "metric_time__day,listing__capacity_latest",
+                "--limit",
+                "3",
+            ],
+        )
+
+    assert resp.exit_code == 0
+
+    assert_object_snapshot_equal(
+        request=request,
+        mf_test_session_state=mf_test_session_state,
+        obj_id="cli_output",
+        obj=resp.output,
+        sql_client=sql_client,
+    )
+
+
+def test_saved_query_explain(  # noqa: D
+    capsys: pytest.CaptureFixture,
+    mf_test_session_state: MetricFlowTestSessionState,
+    cli_runner: MetricFlowCliRunner,
+) -> None:
+    # Disabling capsys to resolve error "ValueError: I/O operation on closed file". Better solution TBD.
+    with capsys.disabled():
+        resp = cli_runner.run(
+            query,
+            args=["--explain", "--saved-query", "p0_booking", "--order", "metric_time__day,listing__capacity_latest"],
+        )
+
+    # Currently difficult to compare explain output due to randomly generated IDs.
+    assert resp.exit_code == 0

metricflow/test/fixtures/semantic_manifest_yamls/simple_manifest/saved_queries.yaml

@@ -0,0 +1,12 @@
+---
+saved_query:
+  name: p0_booking
+  description: Booking-related metrics that are of the highest priority.
+  metrics:
+    - bookings
+    - instant_bookings
+  group_bys:
+    - TimeDimension('metric_time', 'day')
+    - Dimension('listing__capacity_latest')
+  where:
+    - "{{ Dimension('listing__capacity_latest') }} > 3"

metricflow/test/snapshot_utils.py

@@ -167,14 +167,16 @@ def assert_snapshot_text_equal(
     if incomparable_strings_replacement_function is not None:
         snapshot_text = incomparable_strings_replacement_function(snapshot_text)
 
+    # Add a new line at the end of the file so that PRs don't show the "no newline" symbol on Github.
+    if len(snapshot_text) > 1 and snapshot_text[-1] != "\n":
+        snapshot_text = snapshot_text + "\n"
+
     # If we are in overwrite mode, make a new plan:
     if mf_test_session_state.overwrite_snapshots:
         # Create parent directory for the plan text files.
         os.makedirs(os.path.dirname(file_path), exist_ok=True)
         with open(file_path, "w") as snapshot_text_file:
             snapshot_text_file.write(snapshot_text)
-            # Add a new line at the end of the file so that PRSs don't show the "no newline" symbol on Github.
-            snapshot_text_file.write("\n")
 
     # Throw an exception if the plan is not there.
     if not os.path.exists(file_path):
@@ -198,8 +200,7 @@ def assert_snapshot_text_equal(
 
     # Read the existing plan from the file and compare with the actual plan
     with open(file_path, "r") as snapshot_text_file:
-        # Remove the newline that was added from above.
-        expected_snapshot_text = snapshot_text_file.read().rstrip()
+        expected_snapshot_text = snapshot_text_file.read()
 
         if exclude_line_regex:
             # Filter out lines that should be ignored.
@@ -257,6 +258,7 @@ def assert_object_snapshot_equal(  # type: ignore[misc]
     mf_test_session_state: MetricFlowTestSessionState,
     obj_id: str,
     obj: Any,
+    sql_client: Optional[SqlClient] = None,
 ) -> None:
     """For tests to compare large objects, this can be used to snapshot a text representation of the object."""
     assert_snapshot_text_equal(
@@ -266,6 +268,7 @@ def assert_object_snapshot_equal(  # type: ignore[misc]
         snapshot_id=obj_id,
         snapshot_text=pformat_big_objects(obj),
         snapshot_file_extension=".txt",
+        additional_sub_directories_for_snapshots=(sql_client.sql_engine_type.value,) if sql_client else (),
     )
 
 

metricflow/test/snapshots/test_cli.py/str/DuckDB/test_saved_query__cli_output.txt

@@ -0,0 +1,11 @@
+| metric_time__day   |   listing__capacity_latest |   bookings |   instant_bookings |
+|:-------------------|---------------------------:|-----------:|-------------------:|
+| 2019-12-01         |                          5 |          1 |                  0 |
+| 2019-12-18         |                          4 |          4 |                  2 |
+| 2019-12-19         |                          4 |          6 |                  6 |
+| 2019-12-19         |                          5 |          2 |                  0 |
+| 2019-12-20         |                          5 |          2 |                  0 |
+| 2020-01-01         |                          4 |          2 |                  1 |
+| 2020-01-02         |                          4 |          3 |                  3 |
+| 2020-01-02         |                          5 |          1 |                  0 |
+| 2020-01-03         |                          5 |          1 |                  0 |

metricflow/test/snapshots/test_cli.py/str/DuckDB/test_saved_query_with_limit__cli_output.txt

@@ -0,0 +1,5 @@
+| metric_time__day   |   listing__capacity_latest |   bookings |   instant_bookings |
+|:-------------------|---------------------------:|-----------:|-------------------:|
+| 2019-12-01         |                          5 |          1 |                  0 |
+| 2019-12-18         |                          4 |          4 |                  2 |
+| 2019-12-19         |                          4 |          6 |                  6 |

metricflow/test/snapshots/test_cli.py/str/DuckDB/test_saved_query_with_where__cli_output.txt

@@ -0,0 +1,7 @@
+| metric_time__day   |   listing__capacity_latest |   bookings |   instant_bookings |
+|:-------------------|---------------------------:|-----------:|-------------------:|
+| 2019-12-01         |                          5 |          1 |                  0 |
+| 2019-12-19         |                          5 |          2 |                  0 |
+| 2019-12-20         |                          5 |          2 |                  0 |
+| 2020-01-02         |                          5 |          1 |                  0 |
+| 2020-01-03         |                          5 |          1 |                  0 |

metricflow/test/snapshots/test_cli.py/str/Postgres/test_saved_query__cli_output.txt

@@ -0,0 +1,11 @@
+| metric_time__day    |   listing__capacity_latest |   bookings |   instant_bookings |
+|:--------------------|---------------------------:|-----------:|-------------------:|
+| 2019-12-19 00:00:00 |                          4 |          6 |                  6 |
+| 2019-12-18 00:00:00 |                          4 |          4 |                  2 |
+| 2019-12-19 00:00:00 |                          5 |          2 |                  0 |
+| 2020-01-02 00:00:00 |                          4 |          3 |                  3 |
+| 2019-12-01 00:00:00 |                          5 |          1 |                  0 |
+| 2019-12-20 00:00:00 |                          5 |          2 |                  0 |
+| 2020-01-02 00:00:00 |                          5 |          1 |                  0 |
+| 2020-01-01 00:00:00 |                          4 |          2 |                  1 |
+| 2020-01-03 00:00:00 |                          5 |          1 |                  0 |

metricflow/test/snapshots/test_cli.py/str/Postgres/test_saved_query_with_limit__cli_output.txt

@@ -0,0 +1,5 @@
+| metric_time__day    |   listing__capacity_latest |   bookings |   instant_bookings |
+|:--------------------|---------------------------:|-----------:|-------------------:|
+| 2019-12-19 00:00:00 |                          4 |          6 |                  6 |
+| 2019-12-18 00:00:00 |                          4 |          4 |                  2 |
+| 2019-12-19 00:00:00 |                          5 |          2 |                  0 |

metricflow/test/snapshots/test_cli.py/str/Postgres/test_saved_query_with_where__cli_output.txt

@@ -0,0 +1,7 @@
+| metric_time__day    |   listing__capacity_latest |   bookings |   instant_bookings |
+|:--------------------|---------------------------:|-----------:|-------------------:|
+| 2019-12-19 00:00:00 |                          5 |          2 |                  0 |
+| 2019-12-01 00:00:00 |                          5 |          1 |                  0 |
+| 2019-12-20 00:00:00 |                          5 |          2 |                  0 |
+| 2020-01-02 00:00:00 |                          5 |          1 |                  0 |
+| 2020-01-03 00:00:00 |                          5 |          1 |                  0 |