From 6b90b0848f031f32f8c6fa3f6994d0ab839cd5ee Mon Sep 17 00:00:00 2001 From: sroyal-statsig <76536058+sroyal-statsig@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:48:43 -0800 Subject: [PATCH] Add Eval Callback Option (#234) --- statsig/__init__.py | 1 + statsig/feature_gate.py | 15 +++++ statsig/statsig_options.py | 6 ++ statsig/statsig_server.py | 48 +++++++++++++--- tests/test_eval_callback.py | 106 ++++++++++++++++++++++++++++++++++++ 5 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 statsig/feature_gate.py create mode 100644 tests/test_eval_callback.py diff --git a/statsig/__init__.py b/statsig/__init__.py index 28465e3..4b5e3bc 100644 --- a/statsig/__init__.py +++ b/statsig/__init__.py @@ -1,4 +1,5 @@ from .dynamic_config import DynamicConfig +from .feature_gate import FeatureGate from .layer import Layer from .statsig_event import StatsigEvent from .statsig_options import StatsigOptions diff --git a/statsig/feature_gate.py b/statsig/feature_gate.py new file mode 100644 index 0000000..8c01207 --- /dev/null +++ b/statsig/feature_gate.py @@ -0,0 +1,15 @@ +class FeatureGate: + def __init__(self, data, name, rule, group_name=None): + self.value = False if data is None else data + self.name = "" if name is None else name + self.rule_id = "" if rule is None else rule + self.group_name = group_name + + def get_value(self): + """Returns the underlying value of this FeatureGate""" + return self.value + + def get_name(self): + """Returns the name of this FeatureGate""" + return self.name + \ No newline at end of file diff --git a/statsig/statsig_options.py b/statsig/statsig_options.py index 63a2185..b44de32 100644 --- a/statsig/statsig_options.py +++ b/statsig/statsig_options.py @@ -1,4 +1,8 @@ from typing import Optional, Union, Callable + +from .layer import Layer +from .dynamic_config import DynamicConfig +from .feature_gate import FeatureGate from .statsig_errors import StatsigValueError from .interface_data_store import IDataStore from .statsig_environment_tier import StatsigEnvironmentTier @@ -34,6 +38,7 @@ def __init__( custom_logger: Optional[OutputLogger] = None, enable_debug_logs = False, disable_all_logging = False, + evaluation_callback: Optional[Callable[[Union[Layer, DynamicConfig, FeatureGate]], None]] = None, ): self.data_store = data_store self._environment = None @@ -66,6 +71,7 @@ def __init__( self.custom_logger = custom_logger self.enable_debug_logs = enable_debug_logs self.disable_all_logging = disable_all_logging + self.evaluation_callback = evaluation_callback self._set_logging_copy() def get_logging_copy(self): diff --git a/statsig/statsig_server.py b/statsig/statsig_server.py index 9049af9..f530418 100644 --- a/statsig/statsig_server.py +++ b/statsig/statsig_server.py @@ -1,6 +1,7 @@ import dataclasses import threading -from typing import Optional +from typing import Optional, Union +from .feature_gate import FeatureGate from .layer import Layer from .statsig_errors import StatsigNameError, StatsigRuntimeError, StatsigValueError from .statsig_event import StatsigEvent @@ -110,9 +111,23 @@ def _initialize_impl(self, sdk_key: str, options: StatsigOptions): def check_gate(self, user: StatsigUser, gate_name: str, log_exposure=True): def task(): if not self._verify_inputs(user, gate_name): + feature_gate = FeatureGate( + False, + gate_name, + "", + ) + if not self._options.evaluation_callback is None: + self._options.evaluation_callback(feature_gate) return False result = self.__check_gate(user, gate_name, log_exposure) + feature_gate = FeatureGate( + result.boolean_value, + gate_name, + result.rule_id, + result.group_name, + ) + self.safe_eval_callback(feature_gate) return result.boolean_value return self._errorBoundary.capture( @@ -136,15 +151,20 @@ def manually_log_gate_exposure(self, user: StatsigUser, gate_name: str): def get_config(self, user: StatsigUser, config_name: str, log_exposure=True): def task(): if not self._verify_inputs(user, config_name): - return DynamicConfig({}, config_name, "") + dynamicConfig = DynamicConfig({}, config_name, "") + if not self._options.evaluation_callback is None: + self._options.evaluation_callback(dynamicConfig) + return dynamicConfig result = self.__get_config(user, config_name, log_exposure) - return DynamicConfig( + dynamicConfig = DynamicConfig( result.json_value, config_name, result.rule_id, group_name=result.group_name, ) + self.safe_eval_callback(dynamicConfig) + return dynamicConfig return self._errorBoundary.capture( "get_config", @@ -170,14 +190,19 @@ def get_experiment( ): def task(): if not self._verify_inputs(user, experiment_name): - return DynamicConfig({}, experiment_name, "") + dynamicConfig = DynamicConfig({}, experiment_name, "") + if not self._options.evaluation_callback is None: + self._options.evaluation_callback(dynamicConfig) + return dynamicConfig result = self.__get_config(user, experiment_name, log_exposure) - return DynamicConfig( + dynamicConfig = DynamicConfig( result.json_value, experiment_name, result.rule_id, group_name=result.group_name, ) + self.safe_eval_callback(dynamicConfig) + return dynamicConfig return self._errorBoundary.capture( "get_experiment", @@ -201,7 +226,10 @@ def manually_log_experiment_exposure(self, user: StatsigUser, experiment_name: s def get_layer(self, user: StatsigUser, layer_name: str, log_exposure=True) -> Layer: def task(): if not self._verify_inputs(user, layer_name): - return Layer._create(layer_name, {}, "") + layer = Layer._create(layer_name, {}, "") + if not self._options.evaluation_callback is None: + self._options.evaluation_callback(layer) + return layer normal_user = self.__normalize_user(user) result = self._evaluator.get_layer(normal_user, layer_name) @@ -212,7 +240,7 @@ def log_func(layer: Layer, parameter_name: str): normal_user, layer, parameter_name, result ) - return Layer._create( + layer = Layer._create( layer_name, result.json_value, result.rule_id, @@ -220,6 +248,8 @@ def log_func(layer: Layer, parameter_name: str): result.allocated_experiment, log_func, ) + self.safe_eval_callback(layer) + return layer return self._errorBoundary.capture( "get_layer", @@ -244,6 +274,10 @@ def manually_log_layer_parameter_exposure( user, layer, parameter_name, result, is_manual_exposure=True ) + def safe_eval_callback(self, config: Union[FeatureGate, DynamicConfig, Layer]): + if self._options.evaluation_callback is not None: + self._options.evaluation_callback(config) + def log_event(self, event: StatsigEvent): def task(): if not self._initialized: diff --git a/tests/test_eval_callback.py b/tests/test_eval_callback.py new file mode 100644 index 0000000..dd6d064 --- /dev/null +++ b/tests/test_eval_callback.py @@ -0,0 +1,106 @@ +import time +import os +import unittest +import json + +from typing import Optional, Union, Callable + +from unittest.mock import patch +from gzip_helpers import GzipHelpers +from network_stub import NetworkStub +from statsig import statsig, StatsigUser, StatsigOptions, StatsigEvent, StatsigEnvironmentTier, DynamicConfig, Layer, FeatureGate + +with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../testdata/download_config_specs.json')) as r: + CONFIG_SPECS_RESPONSE = r.read() + +_network_stub = NetworkStub("http://test-statsig-e2e") + + +@patch('requests.request', side_effect=_network_stub.mock) +class TestEvalCallback(unittest.TestCase): + _logs = {} + _gateName = "" + _configName = "" + _layerName = "" + + @classmethod + @patch('requests.request', side_effect=_network_stub.mock) + def setUpClass(cls, mock_request): + _network_stub.stub_request_with_value( + "download_config_specs/.*", 200, json.loads(CONFIG_SPECS_RESPONSE)) + _network_stub.stub_request_with_value("list_1", 200, "+7/rrkvF6\n") + _network_stub.stub_request_with_value("get_id_lists", 200, {"list_1": { + "name": "list_1", + "size": 10, + "url": _network_stub.host + "/list_1", + "creationTime": 1, + "fileID": "file_id_1", + }}) + + def log_event_callback(url: str, **kwargs): + cls._logs = GzipHelpers.decode_body(kwargs) + + _network_stub.stub_request_with_function( + "log_event", 202, log_event_callback) + + cls.statsig_user = StatsigUser( + "regular_user_id", email="testuser@statsig.com", private_attributes={"test": 123}) + cls.random_user = StatsigUser("random") + cls._logs = {} + def callback_func(config: Union[DynamicConfig, FeatureGate, Layer]): + if isinstance(config, FeatureGate): + cls._gateName = config.get_name() + if isinstance(config, DynamicConfig): + cls._configName = config.get_name() + if isinstance(config, Layer): + cls._layerName = config.get_name() + + options = StatsigOptions( + api=_network_stub.host, + tier=StatsigEnvironmentTier.development, + disable_diagnostics=True, + evaluation_callback=callback_func) + + statsig.initialize("secret-key", options) + cls.initTime = round(time.time() * 1000) + + @classmethod + def tearDownClass(cls) -> None: + statsig.shutdown() + + # hacky, yet effective. python runs tests in alphabetical order. + def test_a_check_gate(self, mock_request): + statsig.check_gate(self.statsig_user, "always_on_gate"), + self.assertEqual( + self._gateName, + "always_on_gate" + ) + statsig.check_gate(self.statsig_user, "on_for_statsig_email"), + self.assertEqual( + self._gateName, + "on_for_statsig_email" + ) + + def test_b_dynamic_config(self, mock_request): + statsig.get_config(self.statsig_user, "test_config") + self.assertEqual( + self._configName, + "test_config" + ) + + def test_c_experiment(self, mock_request): + statsig.get_experiment(self.statsig_user, "sample_experiment") + self.assertEqual( + self._configName, + "sample_experiment" + ) + + def test_c_experiment(self, mock_request): + config = statsig.get_layer(self.statsig_user, "a_layer") + self.assertEqual( + self._layerName, + "a_layer" + ) + +if __name__ == '__main__': + unittest.main()