From 37c6b41325e6872a86927d906580fd69beb16656 Mon Sep 17 00:00:00 2001 From: Daniel Bachhuber Date: Fri, 13 Dec 2024 06:26:23 -0800 Subject: [PATCH] fix(experiments): Fix flaky trends query runner test (#26898) --- .../test_experiment_trends_query_runner.py | 140 +++++++++++++++--- 1 file changed, 120 insertions(+), 20 deletions(-) diff --git a/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py index 9d72e3810eea6..cc18a27bd702d 100644 --- a/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py +++ b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py @@ -282,7 +282,11 @@ def test_query_runner(self): team=self.team, event="$feature_flag_called", distinct_id=f"user_{variant}_{i}", - properties={feature_flag_property: variant}, + properties={ + "$feature_flag_response": variant, + feature_flag_property: variant, + "$feature_flag": feature_flag.key, + }, ) flush_persons_and_events() @@ -530,7 +534,11 @@ def test_query_runner_with_holdout(self): team=self.team, event="$feature_flag_called", distinct_id=f"user_{variant}_{i}", - properties={feature_flag_property: variant}, + properties={ + "$feature_flag_response": variant, + feature_flag_property: variant, + "$feature_flag": feature_flag.key, + }, ) flush_persons_and_events() @@ -705,7 +713,11 @@ def test_query_runner_with_data_warehouse_series_avg_amount(self): team=self.team, event="$feature_flag_called", distinct_id=f"user_{variant}_{i}", - properties={feature_flag_property: variant}, + properties={ + "$feature_flag_response": variant, + feature_flag_property: variant, + "$feature_flag": feature_flag.key, + }, timestamp=datetime(2023, 1, i + 1), ) @@ -719,21 +731,33 @@ def test_query_runner_with_data_warehouse_series_avg_amount(self): team=self.team, event="$feature_flag_called", distinct_id="user_test_3", - properties={feature_flag_property: "control"}, + properties={ + "$feature_flag_response": "control", + feature_flag_property: "control", + "$feature_flag": feature_flag.key, + }, timestamp=datetime(2023, 1, 3), ) _create_event( team=self.team, event="Some other event", distinct_id="user_test_3", - properties={feature_flag_property: "control"}, + properties={ + "$feature_flag_response": "control", + feature_flag_property: "control", + "$feature_flag": feature_flag.key, + }, timestamp=datetime(2023, 1, 5), ) _create_event( team=self.team, event="$feature_flag_called", distinct_id="user_test_3", - properties={feature_flag_property: "control"}, + properties={ + "$feature_flag_response": "control", + feature_flag_property: "control", + "$feature_flag": feature_flag.key, + }, timestamp=datetime(2023, 1, 9), ) @@ -813,7 +837,12 @@ def test_query_runner_with_data_warehouse_series_no_end_date_and_nested_id(self) team=self.team, event="$feature_flag_called", distinct_id=f"distinct_{variant}_{i}", - properties={feature_flag_property: variant, "$user_id": f"user_{variant}_{i}"}, + properties={ + "$feature_flag_response": variant, + feature_flag_property: variant, + "$feature_flag": feature_flag.key, + "$user_id": f"user_{variant}_{i}", + }, timestamp=datetime(2023, 1, i + 1), ) @@ -827,21 +856,36 @@ def test_query_runner_with_data_warehouse_series_no_end_date_and_nested_id(self) team=self.team, event="$feature_flag_called", distinct_id="distinct_test_3", - properties={feature_flag_property: "control", "$user_id": "user_test_3"}, + properties={ + "$feature_flag_response": "control", + feature_flag_property: "control", + "$feature_flag": feature_flag.key, + "$user_id": "user_test_3", + }, timestamp=datetime(2023, 1, 3), ) _create_event( team=self.team, event="Some other event", distinct_id="distinct_test_3", - properties={feature_flag_property: "control", "$user_id": "user_test_3"}, + properties={ + "$feature_flag_response": "control", + feature_flag_property: "control", + "$feature_flag": feature_flag.key, + "$user_id": "user_test_3", + }, timestamp=datetime(2023, 1, 5), ) _create_event( team=self.team, event="$feature_flag_called", distinct_id="distinct_test_3", - properties={feature_flag_property: "control", "$user_id": "user_test_3"}, + properties={ + "$feature_flag_response": "control", + feature_flag_property: "control", + "$feature_flag": feature_flag.key, + "$user_id": "user_test_3", + }, timestamp=datetime(2023, 1, 9), ) @@ -934,7 +978,11 @@ def test_query_runner_with_data_warehouse_series_expected_query(self): team=self.team, event="$feature_flag_called", distinct_id=f"user_{variant}_{i}", - properties={feature_flag_property: variant}, + properties={ + "$feature_flag_response": variant, + feature_flag_property: variant, + "$feature_flag": feature_flag.key, + }, timestamp=datetime(2023, 1, i + 1), ) @@ -1051,7 +1099,11 @@ def test_query_runner_with_avg_math(self): team=self.team, event="$feature_flag_called", distinct_id=f"user_{variant}_{i}", - properties={feature_flag_property: variant}, + properties={ + "$feature_flag_response": variant, + feature_flag_property: variant, + "$feature_flag": feature_flag.key, + }, timestamp=datetime(2020, 1, i + 1), ) @@ -1146,7 +1198,11 @@ def test_query_runner_standard_flow(self): { "event": "$feature_flag_called", "timestamp": "2020-01-02", - "properties": {ff_property: "control"}, + "properties": { + "$feature_flag_response": "control", + ff_property: "control", + "$feature_flag": feature_flag.key, + }, }, ], "user_control_2": [ @@ -1154,19 +1210,39 @@ def test_query_runner_standard_flow(self): { "event": "$feature_flag_called", "timestamp": "2020-01-02", - "properties": {ff_property: "control"}, + "properties": { + "$feature_flag_response": "control", + ff_property: "control", + "$feature_flag": feature_flag.key, + }, }, ], "user_test_1": [ {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, {"event": "$pageview", "timestamp": "2020-01-03", "properties": {ff_property: "test"}}, {"event": "$pageview", "timestamp": "2020-01-04", "properties": {ff_property: "test"}}, - {"event": "$feature_flag_called", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, + { + "event": "$feature_flag_called", + "timestamp": "2020-01-02", + "properties": { + "$feature_flag_response": "test", + ff_property: "test", + "$feature_flag": feature_flag.key, + }, + }, ], "user_test_2": [ {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, {"event": "$pageview", "timestamp": "2020-01-03", "properties": {ff_property: "test"}}, - {"event": "$feature_flag_called", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, + { + "event": "$feature_flag_called", + "timestamp": "2020-01-02", + "properties": { + "$feature_flag_response": "test", + ff_property: "test", + "$feature_flag": feature_flag.key, + }, + }, ], }, self.team, @@ -1245,7 +1321,11 @@ def test_query_runner_standard_flow_v2_stats(self): { "event": "$feature_flag_called", "timestamp": "2020-01-02", - "properties": {ff_property: "control"}, + "properties": { + "$feature_flag_response": "control", + ff_property: "control", + "$feature_flag": feature_flag.key, + }, }, ], "user_control_2": [ @@ -1253,19 +1333,39 @@ def test_query_runner_standard_flow_v2_stats(self): { "event": "$feature_flag_called", "timestamp": "2020-01-02", - "properties": {ff_property: "control"}, + "properties": { + "$feature_flag_response": "control", + ff_property: "control", + "$feature_flag": feature_flag.key, + }, }, ], "user_test_1": [ {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, {"event": "$pageview", "timestamp": "2020-01-03", "properties": {ff_property: "test"}}, {"event": "$pageview", "timestamp": "2020-01-04", "properties": {ff_property: "test"}}, - {"event": "$feature_flag_called", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, + { + "event": "$feature_flag_called", + "timestamp": "2020-01-02", + "properties": { + "$feature_flag_response": "test", + ff_property: "test", + "$feature_flag": feature_flag.key, + }, + }, ], "user_test_2": [ {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, {"event": "$pageview", "timestamp": "2020-01-03", "properties": {ff_property: "test"}}, - {"event": "$feature_flag_called", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, + { + "event": "$feature_flag_called", + "timestamp": "2020-01-02", + "properties": { + "$feature_flag_response": "test", + ff_property: "test", + "$feature_flag": feature_flag.key, + }, + }, ], }, self.team,