Skip to content

Commit

Permalink
Add Eval Callback Option (#234)
Browse files Browse the repository at this point in the history
  • Loading branch information
sroyal-statsig authored Mar 8, 2024
1 parent 0b455f2 commit 6b90b08
Show file tree
Hide file tree
Showing 5 changed files with 169 additions and 7 deletions.
1 change: 1 addition & 0 deletions statsig/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from .dynamic_config import DynamicConfig
from .feature_gate import FeatureGate
from .layer import Layer
from .statsig_event import StatsigEvent
from .statsig_options import StatsigOptions
Expand Down
15 changes: 15 additions & 0 deletions statsig/feature_gate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
class FeatureGate:
def __init__(self, data, name, rule, group_name=None):
self.value = False if data is None else data
self.name = "" if name is None else name
self.rule_id = "" if rule is None else rule
self.group_name = group_name

def get_value(self):
"""Returns the underlying value of this FeatureGate"""
return self.value

def get_name(self):
"""Returns the name of this FeatureGate"""
return self.name

6 changes: 6 additions & 0 deletions statsig/statsig_options.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
from typing import Optional, Union, Callable

from .layer import Layer
from .dynamic_config import DynamicConfig
from .feature_gate import FeatureGate
from .statsig_errors import StatsigValueError
from .interface_data_store import IDataStore
from .statsig_environment_tier import StatsigEnvironmentTier
Expand Down Expand Up @@ -34,6 +38,7 @@ def __init__(
custom_logger: Optional[OutputLogger] = None,
enable_debug_logs = False,
disable_all_logging = False,
evaluation_callback: Optional[Callable[[Union[Layer, DynamicConfig, FeatureGate]], None]] = None,
):
self.data_store = data_store
self._environment = None
Expand Down Expand Up @@ -66,6 +71,7 @@ def __init__(
self.custom_logger = custom_logger
self.enable_debug_logs = enable_debug_logs
self.disable_all_logging = disable_all_logging
self.evaluation_callback = evaluation_callback
self._set_logging_copy()

def get_logging_copy(self):
Expand Down
48 changes: 41 additions & 7 deletions statsig/statsig_server.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import dataclasses
import threading
from typing import Optional
from typing import Optional, Union
from .feature_gate import FeatureGate
from .layer import Layer
from .statsig_errors import StatsigNameError, StatsigRuntimeError, StatsigValueError
from .statsig_event import StatsigEvent
Expand Down Expand Up @@ -110,9 +111,23 @@ def _initialize_impl(self, sdk_key: str, options: StatsigOptions):
def check_gate(self, user: StatsigUser, gate_name: str, log_exposure=True):
def task():
if not self._verify_inputs(user, gate_name):
feature_gate = FeatureGate(
False,
gate_name,
"",
)
if not self._options.evaluation_callback is None:
self._options.evaluation_callback(feature_gate)
return False

result = self.__check_gate(user, gate_name, log_exposure)
feature_gate = FeatureGate(
result.boolean_value,
gate_name,
result.rule_id,
result.group_name,
)
self.safe_eval_callback(feature_gate)
return result.boolean_value

return self._errorBoundary.capture(
Expand All @@ -136,15 +151,20 @@ def manually_log_gate_exposure(self, user: StatsigUser, gate_name: str):
def get_config(self, user: StatsigUser, config_name: str, log_exposure=True):
def task():
if not self._verify_inputs(user, config_name):
return DynamicConfig({}, config_name, "")
dynamicConfig = DynamicConfig({}, config_name, "")
if not self._options.evaluation_callback is None:
self._options.evaluation_callback(dynamicConfig)
return dynamicConfig

result = self.__get_config(user, config_name, log_exposure)
return DynamicConfig(
dynamicConfig = DynamicConfig(
result.json_value,
config_name,
result.rule_id,
group_name=result.group_name,
)
self.safe_eval_callback(dynamicConfig)
return dynamicConfig

return self._errorBoundary.capture(
"get_config",
Expand All @@ -170,14 +190,19 @@ def get_experiment(
):
def task():
if not self._verify_inputs(user, experiment_name):
return DynamicConfig({}, experiment_name, "")
dynamicConfig = DynamicConfig({}, experiment_name, "")
if not self._options.evaluation_callback is None:
self._options.evaluation_callback(dynamicConfig)
return dynamicConfig
result = self.__get_config(user, experiment_name, log_exposure)
return DynamicConfig(
dynamicConfig = DynamicConfig(
result.json_value,
experiment_name,
result.rule_id,
group_name=result.group_name,
)
self.safe_eval_callback(dynamicConfig)
return dynamicConfig

return self._errorBoundary.capture(
"get_experiment",
Expand All @@ -201,7 +226,10 @@ def manually_log_experiment_exposure(self, user: StatsigUser, experiment_name: s
def get_layer(self, user: StatsigUser, layer_name: str, log_exposure=True) -> Layer:
def task():
if not self._verify_inputs(user, layer_name):
return Layer._create(layer_name, {}, "")
layer = Layer._create(layer_name, {}, "")
if not self._options.evaluation_callback is None:
self._options.evaluation_callback(layer)
return layer

normal_user = self.__normalize_user(user)
result = self._evaluator.get_layer(normal_user, layer_name)
Expand All @@ -212,14 +240,16 @@ def log_func(layer: Layer, parameter_name: str):
normal_user, layer, parameter_name, result
)

return Layer._create(
layer = Layer._create(
layer_name,
result.json_value,
result.rule_id,
result.group_name,
result.allocated_experiment,
log_func,
)
self.safe_eval_callback(layer)
return layer

return self._errorBoundary.capture(
"get_layer",
Expand All @@ -244,6 +274,10 @@ def manually_log_layer_parameter_exposure(
user, layer, parameter_name, result, is_manual_exposure=True
)

def safe_eval_callback(self, config: Union[FeatureGate, DynamicConfig, Layer]):
if self._options.evaluation_callback is not None:
self._options.evaluation_callback(config)

def log_event(self, event: StatsigEvent):
def task():
if not self._initialized:
Expand Down
106 changes: 106 additions & 0 deletions tests/test_eval_callback.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
import time
import os
import unittest
import json

from typing import Optional, Union, Callable

from unittest.mock import patch
from gzip_helpers import GzipHelpers
from network_stub import NetworkStub
from statsig import statsig, StatsigUser, StatsigOptions, StatsigEvent, StatsigEnvironmentTier, DynamicConfig, Layer, FeatureGate

with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../testdata/download_config_specs.json')) as r:
CONFIG_SPECS_RESPONSE = r.read()

_network_stub = NetworkStub("http://test-statsig-e2e")


@patch('requests.request', side_effect=_network_stub.mock)
class TestEvalCallback(unittest.TestCase):
_logs = {}
_gateName = ""
_configName = ""
_layerName = ""

@classmethod
@patch('requests.request', side_effect=_network_stub.mock)
def setUpClass(cls, mock_request):
_network_stub.stub_request_with_value(
"download_config_specs/.*", 200, json.loads(CONFIG_SPECS_RESPONSE))
_network_stub.stub_request_with_value("list_1", 200, "+7/rrkvF6\n")
_network_stub.stub_request_with_value("get_id_lists", 200, {"list_1": {
"name": "list_1",
"size": 10,
"url": _network_stub.host + "/list_1",
"creationTime": 1,
"fileID": "file_id_1",
}})

def log_event_callback(url: str, **kwargs):
cls._logs = GzipHelpers.decode_body(kwargs)

_network_stub.stub_request_with_function(
"log_event", 202, log_event_callback)

cls.statsig_user = StatsigUser(
"regular_user_id", email="[email protected]", private_attributes={"test": 123})
cls.random_user = StatsigUser("random")
cls._logs = {}
def callback_func(config: Union[DynamicConfig, FeatureGate, Layer]):
if isinstance(config, FeatureGate):
cls._gateName = config.get_name()
if isinstance(config, DynamicConfig):
cls._configName = config.get_name()
if isinstance(config, Layer):
cls._layerName = config.get_name()

options = StatsigOptions(
api=_network_stub.host,
tier=StatsigEnvironmentTier.development,
disable_diagnostics=True,
evaluation_callback=callback_func)

statsig.initialize("secret-key", options)
cls.initTime = round(time.time() * 1000)

@classmethod
def tearDownClass(cls) -> None:
statsig.shutdown()

# hacky, yet effective. python runs tests in alphabetical order.
def test_a_check_gate(self, mock_request):
statsig.check_gate(self.statsig_user, "always_on_gate"),
self.assertEqual(
self._gateName,
"always_on_gate"
)
statsig.check_gate(self.statsig_user, "on_for_statsig_email"),
self.assertEqual(
self._gateName,
"on_for_statsig_email"
)

def test_b_dynamic_config(self, mock_request):
statsig.get_config(self.statsig_user, "test_config")
self.assertEqual(
self._configName,
"test_config"
)

def test_c_experiment(self, mock_request):
statsig.get_experiment(self.statsig_user, "sample_experiment")
self.assertEqual(
self._configName,
"sample_experiment"
)

def test_c_experiment(self, mock_request):
config = statsig.get_layer(self.statsig_user, "a_layer")
self.assertEqual(
self._layerName,
"a_layer"
)

if __name__ == '__main__':
unittest.main()

0 comments on commit 6b90b08

Please sign in to comment.