Skip to content

Commit

Permalink
Add CLI tests with snapshots to generate_shapshots.py
Browse files Browse the repository at this point in the history 
The new saved query CLI tests rely on snapshot generation, but
the generate snapshot script will not update them and so our engine
tests are difficult to maintain.

This adds the relevant tests to the update list and generates the
associated snapshot files by way of illustrating that everything works.

We might consider moving to a pytest.mark attribute for the modules
and test cases that rely on snapshots, but for the time being the list
of test paths seems fine.
  • Loading branch information
@tlento
tlento committed Oct 28, 2023
1 parent 9a050f1 commit afbe56b
Show file tree
Hide file tree
Showing 13 changed files with 99 additions and 5 deletions.
12 changes: 7 additions & 5 deletions metricflow/test/generate_snapshots.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,11 @@ def as_configurations(self) -> Sequence[MetricFlowTestConfiguration]: # noqa: D
)


SNAPSHOT_GENERATING_TEST_FILES = (
SNAPSHOT_GENERATING_TESTS = (
"metricflow/test/cli/test_cli.py::test_saved_query",
"metricflow/test/cli/test_cli.py::test_saved_query_with_where",
"metricflow/test/cli/test_cli.py::test_saved_query_with_limit",
"metricflow/test/cli/test_cli.py::test_saved_query_explain",
"metricflow/test/dataflow/builder/test_dataflow_plan_builder.py",
"metricflow/test/dataflow/optimizer/source_scan/test_cm_branch_combiner.py",
"metricflow/test/dataflow/optimizer/source_scan/test_source_scan_optimizer.py",
Expand Down Expand Up @@ -178,15 +182,13 @@ def run_cli() -> None: # noqa: D

credential_sets = MetricFlowTestCredentialSetForAllEngines.parse_raw(credential_sets_json_str)

logger.info(
f"Running the following tests to generate snapshots:\n{pformat_big_objects(SNAPSHOT_GENERATING_TEST_FILES)}"
)
logger.info(f"Running the following tests to generate snapshots:\n{pformat_big_objects(SNAPSHOT_GENERATING_TESTS)}")

for test_configuration in credential_sets.as_configurations:
logger.info(
f"Running tests for {test_configuration.engine} with URL: {test_configuration.credential_set.engine_url}"
)
run_tests(test_configuration, SNAPSHOT_GENERATING_TEST_FILES)
run_tests(test_configuration, SNAPSHOT_GENERATING_TESTS)


if __name__ == "__main__":
Expand Down
11 changes: 11 additions & 0 deletions metricflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
| metric_time__day | listing__capacity_latest | bookings | instant_bookings |
|:--------------------|---------------------------:|-----------:|-------------------:|
| 2019-12-01 00:00:00 | 5 | 1 | 0 |
| 2020-01-01 00:00:00 | 4 | 2 | 1 |
| 2020-01-02 00:00:00 | 4 | 3 | 3 |
| 2020-01-02 00:00:00 | 5 | 1 | 0 |
| 2019-12-18 00:00:00 | 4 | 4 | 2 |
| 2020-01-03 00:00:00 | 5 | 1 | 0 |
| 2019-12-19 00:00:00 | 4 | 6 | 6 |
| 2019-12-19 00:00:00 | 5 | 2 | 0 |
| 2019-12-20 00:00:00 | 5 | 2 | 0 |
5 changes: 5 additions & 0 deletions ...cflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_limit__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
| metric_time__day | listing__capacity_latest | bookings | instant_bookings |
|:--------------------|---------------------------:|-----------:|-------------------:|
| 2019-12-01 00:00:00 | 5 | 1 | 0 |
| 2020-01-01 00:00:00 | 4 | 2 | 1 |
| 2020-01-02 00:00:00 | 4 | 3 | 3 |
7 changes: 7 additions & 0 deletions ...cflow/test/snapshots/test_cli.py/str/BigQuery/test_saved_query_with_where__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
| metric_time__day | listing__capacity_latest | bookings | instant_bookings |
|:--------------------|---------------------------:|-----------:|-------------------:|
| 2019-12-01 00:00:00 | 5 | 1 | 0 |
| 2020-01-02 00:00:00 | 5 | 1 | 0 |
| 2020-01-03 00:00:00 | 5 | 1 | 0 |
| 2019-12-19 00:00:00 | 5 | 2 | 0 |
| 2019-12-20 00:00:00 | 5 | 2 | 0 |
11 changes: 11 additions & 0 deletions metricflow/test/snapshots/test_cli.py/str/Databricks/test_saved_query__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
| metric_time__day | listing__capacity_latest | bookings | instant_bookings |
|:--------------------------|---------------------------:|-----------:|-------------------:|
| 2020-01-02 00:00:00+00:00 | 5 | 1 | 0 |
| 2019-12-19 00:00:00+00:00 | 4 | 6 | 6 |
| 2019-12-01 00:00:00+00:00 | 5 | 1 | 0 |
| 2019-12-20 00:00:00+00:00 | 5 | 2 | 0 |
| 2019-12-19 00:00:00+00:00 | 5 | 2 | 0 |
| 2019-12-18 00:00:00+00:00 | 4 | 4 | 2 |
| 2020-01-02 00:00:00+00:00 | 4 | 3 | 3 |
| 2020-01-03 00:00:00+00:00 | 5 | 1 | 0 |
| 2020-01-01 00:00:00+00:00 | 4 | 2 | 1 |
5 changes: 5 additions & 0 deletions ...low/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_limit__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
| metric_time__day | listing__capacity_latest | bookings | instant_bookings |
|:--------------------------|---------------------------:|-----------:|-------------------:|
| 2020-01-02 00:00:00+00:00 | 5 | 1 | 0 |
| 2019-12-19 00:00:00+00:00 | 4 | 6 | 6 |
| 2019-12-01 00:00:00+00:00 | 5 | 1 | 0 |
7 changes: 7 additions & 0 deletions ...low/test/snapshots/test_cli.py/str/Databricks/test_saved_query_with_where__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
| metric_time__day | listing__capacity_latest | bookings | instant_bookings |
|:--------------------------|---------------------------:|-----------:|-------------------:|
| 2020-01-02 00:00:00+00:00 | 5 | 1 | 0 |
| 2019-12-01 00:00:00+00:00 | 5 | 1 | 0 |
| 2019-12-20 00:00:00+00:00 | 5 | 2 | 0 |
| 2019-12-19 00:00:00+00:00 | 5 | 2 | 0 |
| 2020-01-03 00:00:00+00:00 | 5 | 1 | 0 |
11 changes: 11 additions & 0 deletions metricflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
| metric_time__day | listing__capacity_latest | bookings | instant_bookings |
|:--------------------|---------------------------:|-----------:|-------------------:|
| 2019-12-19 00:00:00 | 5 | 2 | 0 |
| 2019-12-19 00:00:00 | 4 | 6 | 6 |
| 2020-01-03 00:00:00 | 5 | 1 | 0 |
| 2020-01-02 00:00:00 | 5 | 1 | 0 |
| 2019-12-20 00:00:00 | 5 | 2 | 0 |
| 2019-12-01 00:00:00 | 5 | 1 | 0 |
| 2020-01-02 00:00:00 | 4 | 3 | 3 |
| 2020-01-01 00:00:00 | 4 | 2 | 1 |
| 2019-12-18 00:00:00 | 4 | 4 | 2 |
5 changes: 5 additions & 0 deletions ...cflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_limit__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
| metric_time__day | listing__capacity_latest | bookings | instant_bookings |
|:--------------------|---------------------------:|-----------:|-------------------:|
| 2020-01-03 00:00:00 | 5 | 1 | 0 |
| 2020-01-02 00:00:00 | 5 | 1 | 0 |
| 2019-12-20 00:00:00 | 5 | 2 | 0 |
7 changes: 7 additions & 0 deletions ...cflow/test/snapshots/test_cli.py/str/Redshift/test_saved_query_with_where__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
| metric_time__day | listing__capacity_latest | bookings | instant_bookings |
|:--------------------|---------------------------:|-----------:|-------------------:|
| 2019-12-19 00:00:00 | 5 | 2 | 0 |
| 2020-01-03 00:00:00 | 5 | 1 | 0 |
| 2020-01-02 00:00:00 | 5 | 1 | 0 |
| 2019-12-20 00:00:00 | 5 | 2 | 0 |
| 2019-12-01 00:00:00 | 5 | 1 | 0 |
11 changes: 11 additions & 0 deletions metricflow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
| METRIC_TIME__DAY | LISTING__CAPACITY_LATEST | BOOKINGS | INSTANT_BOOKINGS |
|:--------------------|---------------------------:|-----------:|-------------------:|
| 2019-12-01 00:00:00 | 5 | 1 | 0 |
| 2019-12-18 00:00:00 | 4 | 4 | 2 |
| 2019-12-19 00:00:00 | 4 | 6 | 6 |
| 2019-12-19 00:00:00 | 5 | 2 | 0 |
| 2019-12-20 00:00:00 | 5 | 2 | 0 |
| 2020-01-01 00:00:00 | 4 | 2 | 1 |
| 2020-01-02 00:00:00 | 5 | 1 | 0 |
| 2020-01-02 00:00:00 | 4 | 3 | 3 |
| 2020-01-03 00:00:00 | 5 | 1 | 0 |
5 changes: 5 additions & 0 deletions ...flow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_limit__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
| METRIC_TIME__DAY | LISTING__CAPACITY_LATEST | BOOKINGS | INSTANT_BOOKINGS |
|:--------------------|---------------------------:|-----------:|-------------------:|
| 2019-12-01 00:00:00 | 5 | 1 | 0 |
| 2019-12-18 00:00:00 | 4 | 4 | 2 |
| 2019-12-19 00:00:00 | 4 | 6 | 6 |
7 changes: 7 additions & 0 deletions ...flow/test/snapshots/test_cli.py/str/Snowflake/test_saved_query_with_where__cli_output.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
| METRIC_TIME__DAY | LISTING__CAPACITY_LATEST | BOOKINGS | INSTANT_BOOKINGS |
|:--------------------|---------------------------:|-----------:|-------------------:|
| 2019-12-01 00:00:00 | 5 | 1 | 0 |
| 2019-12-19 00:00:00 | 5 | 2 | 0 |
| 2019-12-20 00:00:00 | 5 | 2 | 0 |
| 2020-01-02 00:00:00 | 5 | 1 | 0 |
| 2020-01-03 00:00:00 | 5 | 1 | 0 |

0 comments on commit afbe56b

Please sign in to comment.