From afbe56bee8e41eb710f07aceaade1ef2db91e08c Mon Sep 17 00:00:00 2001 From: tlento Date: Fri, 27 Oct 2023 17:48:07 -0700 Subject: [PATCH] Add CLI tests with snapshots to generate_shapshots.py The new saved query CLI tests rely on snapshot generation, but the generate snapshot script will not update them and so our engine tests are difficult to maintain. This adds the relevant tests to the update list and generates the associated snapshot files by way of illustrating that everything works. We might consider moving to a pytest.mark attribute for the modules and test cases that rely on snapshots, but for the time being the list of test paths seems fine. --- metricflow/test/generate_snapshots.py | 12 +++++++----- .../str/BigQuery/test_saved_query__cli_output.txt | 11 +++++++++++ .../test_saved_query_with_limit__cli_output.txt | 5 +++++ .../test_saved_query_with_where__cli_output.txt | 7 +++++++ .../str/Databricks/test_saved_query__cli_output.txt | 11 +++++++++++ .../test_saved_query_with_limit__cli_output.txt | 5 +++++ .../test_saved_query_with_where__cli_output.txt | 7 +++++++ .../str/Redshift/test_saved_query__cli_output.txt | 11 +++++++++++ .../test_saved_query_with_limit__cli_output.txt | 5 +++++ .../test_saved_query_with_where__cli_output.txt | 7 +++++++ .../str/Snowflake/test_saved_query__cli_output.txt | 11 +++++++++++ .../test_saved_query_with_limit__cli_output.txt | 5 +++++ .../test_saved_query_with_where__cli_output.txt | 7 +++++++ 13 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_limit__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_where__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_limit__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_where__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_limit__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_where__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_limit__cli_output.txt create mode 100644 metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_where__cli_output.txt diff --git a/metricflow/test/generate_snapshots.py b/metricflow/test/generate_snapshots.py index 3ac4446f81..82c5d294cf 100644 --- a/metricflow/test/generate_snapshots.py +++ b/metricflow/test/generate_snapshots.py @@ -97,7 +97,11 @@ def as_configurations(self) -> Sequence[MetricFlowTestConfiguration]: # noqa: D ) -SNAPSHOT_GENERATING_TEST_FILES = ( +SNAPSHOT_GENERATING_TESTS = ( + "metricflow/test/cli/test_cli.py::test_saved_query", + "metricflow/test/cli/test_cli.py::test_saved_query_with_where", + "metricflow/test/cli/test_cli.py::test_saved_query_with_limit", + "metricflow/test/cli/test_cli.py::test_saved_query_explain", "metricflow/test/dataflow/builder/test_dataflow_plan_builder.py", "metricflow/test/dataflow/optimizer/source_scan/test_cm_branch_combiner.py", "metricflow/test/dataflow/optimizer/source_scan/test_source_scan_optimizer.py", @@ -178,15 +182,13 @@ def run_cli() -> None: # noqa: D credential_sets = MetricFlowTestCredentialSetForAllEngines.parse_raw(credential_sets_json_str) - logger.info( - f"Running the following tests to generate snapshots:\n{pformat_big_objects(SNAPSHOT_GENERATING_TEST_FILES)}" - ) + logger.info(f"Running the following tests to generate snapshots:\n{pformat_big_objects(SNAPSHOT_GENERATING_TESTS)}") for test_configuration in credential_sets.as_configurations: logger.info( f"Running tests for {test_configuration.engine} with URL: {test_configuration.credential_set.engine_url}" ) - run_tests(test_configuration, SNAPSHOT_GENERATING_TEST_FILES) + run_tests(test_configuration, SNAPSHOT_GENERATING_TESTS) if __name__ == "__main__": diff --git a/metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query__cli_output.txt new file mode 100644 index 0000000000..30e79e45b0 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query__cli_output.txt @@ -0,0 +1,11 @@ +| metric_time__day | listing__capacity_latest | bookings | instant_bookings | +|:--------------------|---------------------------:|-----------:|-------------------:| +| 2019-12-01 00:00:00 | 5 | 1 | 0 | +| 2020-01-01 00:00:00 | 4 | 2 | 1 | +| 2020-01-02 00:00:00 | 4 | 3 | 3 | +| 2020-01-02 00:00:00 | 5 | 1 | 0 | +| 2019-12-18 00:00:00 | 4 | 4 | 2 | +| 2020-01-03 00:00:00 | 5 | 1 | 0 | +| 2019-12-19 00:00:00 | 4 | 6 | 6 | +| 2019-12-19 00:00:00 | 5 | 2 | 0 | +| 2019-12-20 00:00:00 | 5 | 2 | 0 | diff --git a/metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_limit__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_limit__cli_output.txt new file mode 100644 index 0000000000..c2123f8ea1 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_limit__cli_output.txt @@ -0,0 +1,5 @@ +| metric_time__day | listing__capacity_latest | bookings | instant_bookings | +|:--------------------|---------------------------:|-----------:|-------------------:| +| 2019-12-01 00:00:00 | 5 | 1 | 0 | +| 2020-01-01 00:00:00 | 4 | 2 | 1 | +| 2020-01-02 00:00:00 | 4 | 3 | 3 | diff --git a/metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_where__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_where__cli_output.txt new file mode 100644 index 0000000000..cce16e9717 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_where__cli_output.txt @@ -0,0 +1,7 @@ +| metric_time__day | listing__capacity_latest | bookings | instant_bookings | +|:--------------------|---------------------------:|-----------:|-------------------:| +| 2019-12-01 00:00:00 | 5 | 1 | 0 | +| 2020-01-02 00:00:00 | 5 | 1 | 0 | +| 2020-01-03 00:00:00 | 5 | 1 | 0 | +| 2019-12-19 00:00:00 | 5 | 2 | 0 | +| 2019-12-20 00:00:00 | 5 | 2 | 0 | diff --git a/metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query__cli_output.txt new file mode 100644 index 0000000000..058e8f0608 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query__cli_output.txt @@ -0,0 +1,11 @@ +| metric_time__day | listing__capacity_latest | bookings | instant_bookings | +|:--------------------------|---------------------------:|-----------:|-------------------:| +| 2020-01-02 00:00:00+00:00 | 5 | 1 | 0 | +| 2019-12-19 00:00:00+00:00 | 4 | 6 | 6 | +| 2019-12-01 00:00:00+00:00 | 5 | 1 | 0 | +| 2019-12-20 00:00:00+00:00 | 5 | 2 | 0 | +| 2019-12-19 00:00:00+00:00 | 5 | 2 | 0 | +| 2019-12-18 00:00:00+00:00 | 4 | 4 | 2 | +| 2020-01-02 00:00:00+00:00 | 4 | 3 | 3 | +| 2020-01-03 00:00:00+00:00 | 5 | 1 | 0 | +| 2020-01-01 00:00:00+00:00 | 4 | 2 | 1 | diff --git a/metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_limit__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_limit__cli_output.txt new file mode 100644 index 0000000000..53ccdfa167 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_limit__cli_output.txt @@ -0,0 +1,5 @@ +| metric_time__day | listing__capacity_latest | bookings | instant_bookings | +|:--------------------------|---------------------------:|-----------:|-------------------:| +| 2020-01-02 00:00:00+00:00 | 5 | 1 | 0 | +| 2019-12-19 00:00:00+00:00 | 4 | 6 | 6 | +| 2019-12-01 00:00:00+00:00 | 5 | 1 | 0 | diff --git a/metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_where__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_where__cli_output.txt new file mode 100644 index 0000000000..06daf91ce8 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_where__cli_output.txt @@ -0,0 +1,7 @@ +| metric_time__day | listing__capacity_latest | bookings | instant_bookings | +|:--------------------------|---------------------------:|-----------:|-------------------:| +| 2020-01-02 00:00:00+00:00 | 5 | 1 | 0 | +| 2019-12-01 00:00:00+00:00 | 5 | 1 | 0 | +| 2019-12-20 00:00:00+00:00 | 5 | 2 | 0 | +| 2019-12-19 00:00:00+00:00 | 5 | 2 | 0 | +| 2020-01-03 00:00:00+00:00 | 5 | 1 | 0 | diff --git a/metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query__cli_output.txt new file mode 100644 index 0000000000..beccedd97c --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query__cli_output.txt @@ -0,0 +1,11 @@ +| metric_time__day | listing__capacity_latest | bookings | instant_bookings | +|:--------------------|---------------------------:|-----------:|-------------------:| +| 2019-12-19 00:00:00 | 5 | 2 | 0 | +| 2019-12-19 00:00:00 | 4 | 6 | 6 | +| 2020-01-03 00:00:00 | 5 | 1 | 0 | +| 2020-01-02 00:00:00 | 5 | 1 | 0 | +| 2019-12-20 00:00:00 | 5 | 2 | 0 | +| 2019-12-01 00:00:00 | 5 | 1 | 0 | +| 2020-01-02 00:00:00 | 4 | 3 | 3 | +| 2020-01-01 00:00:00 | 4 | 2 | 1 | +| 2019-12-18 00:00:00 | 4 | 4 | 2 | diff --git a/metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_limit__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_limit__cli_output.txt new file mode 100644 index 0000000000..f5456036e3 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_limit__cli_output.txt @@ -0,0 +1,5 @@ +| metric_time__day | listing__capacity_latest | bookings | instant_bookings | +|:--------------------|---------------------------:|-----------:|-------------------:| +| 2020-01-03 00:00:00 | 5 | 1 | 0 | +| 2020-01-02 00:00:00 | 5 | 1 | 0 | +| 2019-12-20 00:00:00 | 5 | 2 | 0 | diff --git a/metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_where__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_where__cli_output.txt new file mode 100644 index 0000000000..44f958aec2 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_where__cli_output.txt @@ -0,0 +1,7 @@ +| metric_time__day | listing__capacity_latest | bookings | instant_bookings | +|:--------------------|---------------------------:|-----------:|-------------------:| +| 2019-12-19 00:00:00 | 5 | 2 | 0 | +| 2020-01-03 00:00:00 | 5 | 1 | 0 | +| 2020-01-02 00:00:00 | 5 | 1 | 0 | +| 2019-12-20 00:00:00 | 5 | 2 | 0 | +| 2019-12-01 00:00:00 | 5 | 1 | 0 | diff --git a/metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query__cli_output.txt new file mode 100644 index 0000000000..26caef8593 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query__cli_output.txt @@ -0,0 +1,11 @@ +| METRIC_TIME__DAY | LISTING__CAPACITY_LATEST | BOOKINGS | INSTANT_BOOKINGS | +|:--------------------|---------------------------:|-----------:|-------------------:| +| 2019-12-01 00:00:00 | 5 | 1 | 0 | +| 2019-12-18 00:00:00 | 4 | 4 | 2 | +| 2019-12-19 00:00:00 | 4 | 6 | 6 | +| 2019-12-19 00:00:00 | 5 | 2 | 0 | +| 2019-12-20 00:00:00 | 5 | 2 | 0 | +| 2020-01-01 00:00:00 | 4 | 2 | 1 | +| 2020-01-02 00:00:00 | 5 | 1 | 0 | +| 2020-01-02 00:00:00 | 4 | 3 | 3 | +| 2020-01-03 00:00:00 | 5 | 1 | 0 | diff --git a/metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_limit__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_limit__cli_output.txt new file mode 100644 index 0000000000..49ab821a22 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_limit__cli_output.txt @@ -0,0 +1,5 @@ +| METRIC_TIME__DAY | LISTING__CAPACITY_LATEST | BOOKINGS | INSTANT_BOOKINGS | +|:--------------------|---------------------------:|-----------:|-------------------:| +| 2019-12-01 00:00:00 | 5 | 1 | 0 | +| 2019-12-18 00:00:00 | 4 | 4 | 2 | +| 2019-12-19 00:00:00 | 4 | 6 | 6 | diff --git a/metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_where__cli_output.txt b/metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_where__cli_output.txt new file mode 100644 index 0000000000..693e9c3f74 --- /dev/null +++ b/metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_where__cli_output.txt @@ -0,0 +1,7 @@ +| METRIC_TIME__DAY | LISTING__CAPACITY_LATEST | BOOKINGS | INSTANT_BOOKINGS | +|:--------------------|---------------------------:|-----------:|-------------------:| +| 2019-12-01 00:00:00 | 5 | 1 | 0 | +| 2019-12-19 00:00:00 | 5 | 2 | 0 | +| 2019-12-20 00:00:00 | 5 | 2 | 0 | +| 2020-01-02 00:00:00 | 5 | 1 | 0 | +| 2020-01-03 00:00:00 | 5 | 1 | 0 |