From c6a6f9cdc3cc6f6cb990113d831582923f780242 Mon Sep 17 00:00:00 2001 From: adisun Date: Tue, 13 Jun 2023 12:54:51 -0400 Subject: [PATCH 01/25] add ttl and caching support in vault provider --- gestalt/__init__.py | 28 +++++++++++++++++++++++----- gestalt/provider.py | 7 +++++++ gestalt/vault.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/gestalt/__init__.py b/gestalt/__init__.py index 53331f9..277c67c 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -166,9 +166,9 @@ def build_config(self) -> None: sep=self.__delim_char) self.__parse_dictionary_keys(self.__conf_data) - self.__conf_data = self.__interpolate_keys(self.__conf_data) + #self.__conf_data = self.__interpolate_keys(self.__conf_data) self.__parse_dictionary_keys(self.__conf_sets) - self.__conf_sets = self.__interpolate_keys(self.__conf_sets) + #self.__conf_sets = self.__interpolate_keys(self.__conf_sets) def __parse_dictionary_keys( self, dictionary: Dict[str, Union[List[Any], str, int, bool, float]] @@ -444,11 +444,29 @@ def __get( f'The environment variable {e_key} could not be converted to type {t}: {e}' ) if key in self.__conf_data: - if not isinstance(self.__conf_data[key], t): + print(f"KEY {key} IN CONF DATA") + val = self.__conf_data[key] + print(f"THE VAL {val}") + regex_search = self.regex_pattern.search(val) + if regex_search is not None: + path = regex_search.group(2) + filter_ = regex_search.group(3) + for provider in self.providers.values(): + if val.startswith(provider.scheme): + interpolated_val = provider.get(key=val, + path=path, + filter=filter_) + break + else: + raise TypeError(f"No scheme found for path {path}.") + + else: + interpolated_val = val + if not isinstance(interpolated_val, t): raise TypeError( - f'The requested key of {key} is not of type {t} (it is {type(self.__conf_data[key])})' + f'Given set key is not of type {t}, but of type {type(val)}' ) - return self.__conf_data[key] + return interpolated_val if default: return default if key in self.__conf_defaults: diff --git a/gestalt/provider.py b/gestalt/provider.py index 434baff..b15f273 100644 --- a/gestalt/provider.py +++ b/gestalt/provider.py @@ -19,3 +19,10 @@ def get(self, key: str, path: str, filter: str) -> Any: """Abstract method to get a value from the provider """ pass + + @property + @abstractmethod + def scheme(self) -> str: + """Returns scheme of provider + """ + pass diff --git a/gestalt/vault.py b/gestalt/vault.py index 574aa36..2a9dee9 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -1,8 +1,9 @@ +from datetime import datetime, timedelta from time import sleep from gestalt.provider import Provider import requests from jsonpath_ng import parse # type: ignore -from typing import Optional, Tuple, Any +from typing import Optional, Tuple, Any, Dict import hvac # type: ignore import asyncio import os @@ -18,7 +19,8 @@ def __init__(self, jwt: Optional[str] = None, url: Optional[str] = os.environ.get("VAULT_ADDR"), token: Optional[str] = os.environ.get("VAULT_TOKEN"), - verify: Optional[bool] = True) -> None: + verify: Optional[bool] = True, + scheme: str = "ref+vault://") -> None: """Initialized vault client and authenticates vault Args: @@ -28,6 +30,7 @@ def __init__(self, auth_config (HVAC_ClientAuthentication): authenticates the initialized vault client with role and jwt string from kubernetes """ + self._scheme: str = scheme self.dynamic_token_queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=0) self.kubes_token_queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=0) @@ -35,6 +38,8 @@ def __init__(self, token=token, cert=cert, verify=verify) + self._secret_expiry_times: Dict[str, datetime] = dict() + self._secret_values: Dict[str, Any] = dict() try: self.vault_client.is_authenticated() @@ -82,6 +87,16 @@ def get(self, key: str, path: str, filter: str) -> Any: Returns: secret (str): secret """ + + # if the key has been read before and is not a TTL secret + if key in self._secret_values and key not in self._secret_expiry_times: + print(f"Found key {key} in cache with no TTL. Not going to Vault.") + return self._secret_values[key] + + # if the secret can expire but hasn't expired yet + if key in self._secret_expiry_times and not self._is_secret_expired(key): + print(f"Found unexpired TTL key {key}. Not going to Vault.") + return self._secret_values[key] try: response = self.vault_client.read(path) if response is None: @@ -109,8 +124,28 @@ def get(self, key: str, path: str, filter: str) -> Any: returned_value_from_secret = match[0].value if returned_value_from_secret == "": raise RuntimeError("Gestalt Error: Empty secret!") + + self._secret_values[key] = returned_value_from_secret + + if "ttl" in requested_data: + self._set_secrets_ttl(requested_data, key) return returned_value_from_secret + def _is_secret_expired(self, key: str) -> bool: + now = datetime.now() + secret_expires_dt = self._secret_expiry_times[key] + is_expired = now >= secret_expires_dt + if is_expired: + logger.debug(f"TTL key {key} found expired.") + return is_expired + + def _set_secrets_ttl(self, requested_data: Dict, key: str) -> None: + last_vault_rotation_str = requested_data["last_vault_rotation"].split(".")[0] # to the nearest second + last_vault_rotation_dt = datetime.strptime(last_vault_rotation_str, '%Y-%m-%dT%H:%M:%S') + ttl = requested_data["ttl"] + secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) + self._secret_expiry_times[key] = secret_expires_dt + async def worker(self, token_queue: Any) -> None: """ Worker function to renew lease on expiry @@ -138,3 +173,7 @@ async def worker(self, token_queue: Any) -> None: "Gestalt Error: Gestalt couldn't connect to Vault") except Exception as err: raise RuntimeError(f"Gestalt Error: {err}") + + @property + def scheme(self) -> str: + return self._scheme From 72d465aa3f0a72481608af12af0b8cf856f8a941 Mon Sep 17 00:00:00 2001 From: adisun Date: Tue, 13 Jun 2023 12:58:02 -0400 Subject: [PATCH 02/25] formatting --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 2a9dee9..6ab7cb0 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -136,7 +136,7 @@ def _is_secret_expired(self, key: str) -> bool: secret_expires_dt = self._secret_expiry_times[key] is_expired = now >= secret_expires_dt if is_expired: - logger.debug(f"TTL key {key} found expired.") + print(f"TTL key {key} found expired.") return is_expired def _set_secrets_ttl(self, requested_data: Dict, key: str) -> None: From f761dc67e6885cbcf19735d0268d413b4b34cfbc Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 16:27:35 -0400 Subject: [PATCH 03/25] add vault cache unit tests --- gestalt/__init__.py | 153 ++++++++++++++++------------- gestalt/provider.py | 4 +- gestalt/utils.py | 17 ++++ gestalt/vault.py | 17 +++- setup.py | 2 +- tests/conftest.py | 88 +++++++++++++++++ tests/test_gestalt.py | 124 +++-------------------- tests/test_vault.py | 29 ++++++ tests/testvault/testsfdynamic.json | 2 +- 9 files changed, 247 insertions(+), 189 deletions(-) create mode 100644 gestalt/utils.py create mode 100644 tests/conftest.py create mode 100644 tests/test_vault.py diff --git a/gestalt/__init__.py b/gestalt/__init__.py index 277c67c..b5d8296 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -2,12 +2,14 @@ from gestalt.provider import Provider import os import glob -import collections.abc as collections + from typing import Dict, List, Type, Union, Optional, MutableMapping, Text, Any import yaml import re import json +from .utils import flatten + def merge_into( a: Dict[Text, Union[List[Any], Text, int, bool, float]], @@ -49,21 +51,6 @@ def __init__(self) -> None: self.regex_pattern = re.compile( r"^ref\+([^\+]*)://([^(\+)]+)\#([^\+]+)?$") - def __flatten( - self, - d: MutableMapping[Text, Any], - parent_key: str = '', - sep: str = '.' - ) -> Dict[Text, Union[List[Any], Text, int, bool, float]]: - items: List[Any] = [] - for k, v in d.items(): - new_key = parent_key + sep + k if parent_key else k - if isinstance(v, collections.MutableMapping): - items.extend(self.__flatten(v, new_key, sep=sep).items()) - else: - items.append((new_key, v)) - return dict(items) - def add_config_path(self, path: str) -> None: """Adds a path to read configs from. @@ -162,13 +149,13 @@ def build_config(self) -> None: f'File {f} is marked as ".yaml" but cannot be read as such: {e}' ) - self.__conf_data = self.__flatten(self.__conf_data, + self.__conf_data = flatten(self.__conf_data, sep=self.__delim_char) self.__parse_dictionary_keys(self.__conf_data) - #self.__conf_data = self.__interpolate_keys(self.__conf_data) + # self.__conf_data = self.__interpolate_keys(self.__conf_data) self.__parse_dictionary_keys(self.__conf_sets) - #self.__conf_sets = self.__interpolate_keys(self.__conf_sets) + # self.__conf_sets = self.__interpolate_keys(self.__conf_sets) def __parse_dictionary_keys( self, dictionary: Dict[str, Union[List[Any], str, int, bool, float]] @@ -223,7 +210,7 @@ def __interpolate_keys( filter=m.group(3)) dictionary.update({config_key: secret}) - dictionary = self.__flatten(dictionary, sep=self.__delim_char) + dictionary = flatten(dictionary, sep=self.__delim_char) return dictionary def auto_env(self) -> None: @@ -425,57 +412,19 @@ def __get( raise TypeError(f'Given key is not of string type') if default and not isinstance(default, t): raise TypeError( - f'Provided default is of incorrect type {type(default)}, it should be of type {t}' + f'Provided default is of incorrect type {type(default)}, it should be of type {t}' ) - if key in self.__conf_sets: - val = self.__conf_sets[key] - if not isinstance(val, t): - raise TypeError( - f'Given set key is not of type {t}, but of type {type(val)}' - ) - return val - if self.__use_env: - e_key = key.upper().replace(self.__delim_char, '_') - if e_key in os.environ: - try: - return t(os.environ[e_key]) - except ValueError as e: - raise TypeError( - f'The environment variable {e_key} could not be converted to type {t}: {e}' - ) - if key in self.__conf_data: - print(f"KEY {key} IN CONF DATA") - val = self.__conf_data[key] - print(f"THE VAL {val}") - regex_search = self.regex_pattern.search(val) - if regex_search is not None: - path = regex_search.group(2) - filter_ = regex_search.group(3) - for provider in self.providers.values(): - if val.startswith(provider.scheme): - interpolated_val = provider.get(key=val, - path=path, - filter=filter_) - break - else: - raise TypeError(f"No scheme found for path {path}.") + split_keys = key.split(self.__delim_char) + consider_keys = list() + print(f"CONF DATA {self.__conf_data}") + print(f"CONF SETS {self.__conf_sets}") + for split_key in split_keys: + consider_keys.append(split_key) + joined_key = ".".join(consider_keys) + config_val = self._get_config_for_key(key=key, key_to_search=joined_key, default=default, object_type=t) + if config_val is not None: + return config_val - else: - interpolated_val = val - if not isinstance(interpolated_val, t): - raise TypeError( - f'Given set key is not of type {t}, but of type {type(val)}' - ) - return interpolated_val - if default: - return default - if key in self.__conf_defaults: - val = self.__conf_defaults[key] - if not isinstance(val, t): - raise TypeError( - f'Given default set key is not of type {t}, but of type {type(val)}' - ) - return val raise ValueError( f'Given key {key} is not in any configuration and no default is provided' ) @@ -615,3 +564,69 @@ def dump(self) -> Text: ret.update(self.__conf_data) ret.update(self.__conf_sets) return str(json.dumps(ret, indent=4)) + + def _get_config_for_key(self, + key: str, + key_to_search, + default: Optional[Union[str, int, float, bool,List[Any]]], + object_type: Type[Union[str, int, float, bool, List[Any]]]) -> Optional[Union[str, int, float, bool, List[Any]]]: + if key_to_search in self.__conf_sets: + val = self.__conf_sets[key_to_search] + if not isinstance(val, object_type): + raise TypeError( + f'Given set key is not of type {object_type}, but of type {type(val)}' + ) + return val + if self.__use_env: + e_key = key_to_search.upper().replace(self.__delim_char, '_') + if e_key in os.environ: + try: + return object_type(os.environ[e_key]) + except ValueError as e: + raise TypeError( + f'The environment variable {e_key} could not be converted to type {object_type}: {e}' + ) + + if key_to_search in self.__conf_data: + print(f"KEY {key_to_search} IN CONF DATA") + val = self.__conf_data[key_to_search] + print(f"THE VAL {val}") + for provider in self.providers.values(): + if val.startswith(provider.scheme): + regex_search = self.regex_pattern.search(val) + if regex_search is not None: + path = regex_search.group(2) + filter_ = regex_search.group(3) + print(f"ORIGINAL KEY {key} cur key {key_to_search}") + remainder_filter = key[len(key_to_search):] + if len(remainder_filter) > 1: + print("IN REMAINDER KEYS") + if filter_ is not None: + filter_ = f".{filter_}{remainder_filter}" + + else: + filter_ = remainder_filter + + print(f"filter is {filter_}") + interpolated_val = provider.get(key=val, + path=path, + filter=filter_, + sep=self.__delim_char) + break + else: + interpolated_val = val + print(f"{interpolated_val=}") + if not isinstance(interpolated_val, object_type): + raise TypeError( + f'Given set key is not of type {object_type}, but of type {type(interpolated_val)}' + ) + return interpolated_val + if default: + return default + if key_to_search in self.__conf_defaults: + val = self.__conf_defaults[key_to_search] + if not isinstance(val, object_type): + raise TypeError( + f'Given default set key is not of type {object_type}, but of type {type(val)}' + ) + return val diff --git a/gestalt/provider.py b/gestalt/provider.py index b15f273..45e18ff 100644 --- a/gestalt/provider.py +++ b/gestalt/provider.py @@ -1,5 +1,5 @@ from abc import ABCMeta, abstractmethod -from typing import Tuple, Dict, Any +from typing import Tuple, Dict, Any, Optional class Provider(metaclass=ABCMeta): @@ -15,7 +15,7 @@ def __init__(self, *args: Tuple[Any], **kwargs: Dict[Any, Any]): pass @abstractmethod - def get(self, key: str, path: str, filter: str) -> Any: + def get(self, key: str, path: str, filter: str, sep: Optional[str]) -> Any: """Abstract method to get a value from the provider """ pass diff --git a/gestalt/utils.py b/gestalt/utils.py new file mode 100644 index 0000000..892f082 --- /dev/null +++ b/gestalt/utils.py @@ -0,0 +1,17 @@ +from typing import MutableMapping, Text, Any, Union, Dict, List +import collections.abc as collections + + +def flatten( + d: MutableMapping[Text, Any], + parent_key: str = '', + sep: str = '.' +) -> Dict[Text, Union[List[Any], Text, int, bool, float]]: + items: List[Any] = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, collections.MutableMapping): + items.extend(flatten(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) \ No newline at end of file diff --git a/gestalt/vault.py b/gestalt/vault.py index 6ab7cb0..862be45 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -11,6 +11,12 @@ from retry import retry +def _get_nested_key(requested_data: Dict, key: str, sep: str) -> Any: + for nested_key in key.split(sep): + requested_data = requested_data[nested_key] + return requested_data + + class Vault(Provider): @retry(exceptions=RuntimeError, delay=2, tries=5) # type: ignore def __init__(self, @@ -78,16 +84,16 @@ def __init__(self, kubernetes_ttl_renew.start() @retry(RuntimeError, delay=3, tries=3) # type: ignore - def get(self, key: str, path: str, filter: str) -> Any: + def get(self, key: str, path: str, filter: str, sep: Optional[str] = ".") -> Any: """Gets secret from vault Args: key (str): key to get secret from path (str): path to secret filter (str): filter to apply to secret + sep (str): delimiter used for flattening Returns: secret (str): secret """ - # if the key has been read before and is not a TTL secret if key in self._secret_values and key not in self._secret_expiry_times: print(f"Found key {key} in cache with no TTL. Not going to Vault.") @@ -97,6 +103,7 @@ def get(self, key: str, path: str, filter: str) -> Any: if key in self._secret_expiry_times and not self._is_secret_expired(key): print(f"Found unexpired TTL key {key}. Not going to Vault.") return self._secret_values[key] + try: response = self.vault_client.read(path) if response is None: @@ -115,6 +122,8 @@ def get(self, key: str, path: str, filter: str) -> Any: except Exception as err: raise RuntimeError(f"Gestalt Error: {err}") if filter is None: + # if len(key.split(sep)) > 1: + # return _get_nested_key(requested_data, key, sep) return requested_data secret = requested_data jsonpath_expression = parse(f"${filter}") @@ -126,9 +135,11 @@ def get(self, key: str, path: str, filter: str) -> Any: raise RuntimeError("Gestalt Error: Empty secret!") self._secret_values[key] = returned_value_from_secret - if "ttl" in requested_data: self._set_secrets_ttl(requested_data, key) + + print(f"THE RETURNED VALUE FROM SECRET {returned_value_from_secret}") + return returned_value_from_secret def _is_secret_expired(self, key: str) -> bool: diff --git a/setup.py b/setup.py index 83ee1cd..00bd948 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def readme(): reqs_list = list(map(lambda x: x.rstrip(), reqs)) setup(name='gestalt-cfg', - version='3.2.0', + version='3.3.0', description='A sensible configuration library for Python', long_description=readme(), long_description_content_type="text/markdown", diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..52379be --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,88 @@ +from unittest.mock import patch, Mock +import pytest +import hvac +import os +import requests + + +class MockSession(requests.Session): + def request(self, *_, **__): + resp = { + 'request_id': '230f5e67-e55d-bdae-bd24-c7bc13c1a3e9', + 'lease_id': '', + 'renewable': False, + 'lease_duration': 0, + 'data': { + 'last_vault_rotation': '2023-05-31T14:24:41.724285249Z', + 'password': 'foo', + 'rotation_period': 60, + 'ttl': 0, + 'username': 'foo' + }, + 'wrap_info': None, + 'warnings': None, + 'auth': None + } + return MockResponse(resp, 200) + + +class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + self.ok = True + + def json(self): + return self.json_data + + +@pytest.fixture +def mock_db_role_request(mocker): + mocker.patch("requests.Session", MockSession) + + +@pytest.fixture(scope="function") +def secret_setup(): + client = hvac.Client() + client.secrets.kv.v2.create_or_update_secret( + path="test", secret=dict(test_secret="test_secret_password")) + + +@pytest.fixture(scope="function") +def incorrect_env_setup(): + os.environ['VAULT_ADDR'] = "" + + +@pytest.fixture(scope="function") +def mount_setup(): + client = hvac.Client() + secret_engines_list = client.sys.list_mounted_secrets_engines( + )['data'].keys() + if "test-mount/" in secret_engines_list: + client.sys.disable_secrets_engine(path="test-mount") + client.sys.enable_secrets_engine(backend_type="kv", path="test-mount") + client.secrets.kv.v2.create_or_update_secret( + mount_point="test-mount", + path="test", + secret=dict(test_mount="test_mount_password")) + + +@pytest.fixture(scope="function") +def nested_setup(): + client = hvac.Client() + client.secrets.kv.v2.create_or_update_secret( + path="testnested", secret=dict(slack={"token": "random-token"})) + + +@pytest.fixture +def mock_vault_workers(): + mock_dynamic_renew = Mock() + mock_k8s_renew = Mock() + patch("gestalt.vault.Thread", + side_effect=[mock_dynamic_renew, mock_k8s_renew]).start() + return (mock_dynamic_renew, mock_k8s_renew) + + +@pytest.fixture +def mock_vault_k8s_auth(): + return patch("gestalt.vault.hvac.api.auth_methods.Kubernetes").start() diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index b712102..c399231 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -9,43 +9,6 @@ import os import gestalt import hvac -import requests - - -class MockSession(requests.Session): - def request(self, *_, **__): - resp = { - 'request_id': '230f5e67-e55d-bdae-bd24-c7bc13c1a3e9', - 'lease_id': '', - 'renewable': False, - 'lease_duration': 0, - 'data': { - 'last_vault_rotation': '2023-05-31T14:24:41.724285249Z', - 'password': 'foo', - 'rotation_period': 60, - 'ttl': 0, - 'username': 'foo' - }, - 'wrap_info': None, - 'warnings': None, - 'auth': None - } - return MockResponse(resp, 200) - - -class MockResponse: - def __init__(self, json_data, status_code): - self.json_data = json_data - self.status_code = status_code - self.ok = True - - def json(self): - return self.json_data - - -@pytest.fixture -def mock_db_role_request(mocker): - mocker.patch("requests.Session", MockSession) # Testing member function @@ -515,30 +478,11 @@ def test_set_default_bad_type_set_config(): assert 'Set config has' in terr -# Vault testing -@pytest.fixture(scope="function") -def env_setup(): - os.environ['VAULT_ADDR'] = "http://localhost:8200" - os.environ['VAULT_TOKEN'] = "myroot" - - -def test_vault_setup(env_setup): +def test_vault_setup(): vault = Vault(role=None, jwt=None) assert vault.vault_client.is_authenticated() is True -@pytest.fixture(scope="function") -def incorrect_env_setup(): - os.environ['VAULT_ADDR'] = "" - - -@pytest.fixture(scope="function") -def secret_setup(env_setup): - client = hvac.Client() - client.secrets.kv.v2.create_or_update_secret( - path="test", secret=dict(test_secret="test_secret_password")) - - def test_vault_interpolation(secret_setup): g = gestalt.Gestalt() g.add_config_file("./tests/testvault/testcorrect.json") @@ -549,21 +493,7 @@ def test_vault_interpolation(secret_setup): assert secret == "test_secret_password" -@pytest.fixture(scope="function") -def mount_setup(env_setup): - client = hvac.Client() - secret_engines_list = client.sys.list_mounted_secrets_engines( - )['data'].keys() - if "test-mount/" in secret_engines_list: - client.sys.disable_secrets_engine(path="test-mount") - client.sys.enable_secrets_engine(backend_type="kv", path="test-mount") - client.secrets.kv.v2.create_or_update_secret( - mount_point="test-mount", - path="test", - secret=dict(test_mount="test_mount_password")) - - -def test_vault_mount_path(env_setup, mount_setup): +def test_vault_mount_path(mount_setup): g = gestalt.Gestalt() g.add_config_file("./tests/testvault/testmount.json") g.configure_provider("vault", Vault(role=None, jwt=None)) @@ -572,36 +502,16 @@ def test_vault_mount_path(env_setup, mount_setup): assert secret == "test_mount_password" -def test_vault_incorrect_path(env_setup, mount_setup): +def test_vault_incorrect_path(mount_setup): g = gestalt.Gestalt() g.add_config_file("./tests/testvault/testincorrectmount.json") g.configure_provider("vault", Vault(role=None, jwt=None)) with pytest.raises(RuntimeError): g.build_config() + g.get_string("test_mount") -@pytest.fixture(scope="function") -def mount_setup(env_setup): - client = hvac.Client() - secret_engines_list = client.sys.list_mounted_secrets_engines( - )['data'].keys() - if "test-mount/" in secret_engines_list: - client.sys.disable_secrets_engine(path="test-mount") - client.sys.enable_secrets_engine(backend_type="kv", path="test-mount") - client.secrets.kv.v2.create_or_update_secret( - mount_point="test-mount", - path="test", - secret=dict(test_mount="test_mount_password")) - - -@pytest.fixture(scope="function") -def nested_setup(env_setup): - client = hvac.Client() - client.secrets.kv.v2.create_or_update_secret( - path="testnested", secret=dict(slack={"token": "random-token"})) - - -def test_nest_key_for_vault(env_setup, nested_setup): +def test_nest_key_for_vault(nested_setup, secret_setup): g = gestalt.Gestalt() g.add_config_file("./tests/testvault/testnested.json") g.configure_provider("vault", Vault(role=None, jwt=None)) @@ -612,37 +522,23 @@ def test_nest_key_for_vault(env_setup, nested_setup): assert secret_slack == "random-token" -def test_read_no_nest_db_role(env_setup, mock_db_role_request): +def test_read_no_nest_db_role(mock_db_role_request): g = gestalt.Gestalt() g.add_config_file("./tests/testvault/testsfdynamic.json") g.configure_provider("vault", Vault(role=None, jwt=None)) g.build_config() - secret_username = g.get_string("snowflake.username") + secret_username = g.get_string("username") assert secret_username == "foo" -def test_set_vault_key(env_setup, nested_setup): +def test_set_vault_key(nested_setup): g = gestalt.Gestalt() g.configure_provider("vault", Vault(role=None, jwt=None)) g.set_string(key="test", value="ref+vault://secret/data/testnested#.slack.token") g.build_config() secret = g.get_string("test") - assert secret == "random-token" - - -@pytest.fixture -def mock_vault_workers(): - mock_dynamic_renew = Mock() - mock_k8s_renew = Mock() - patch("gestalt.vault.Thread", - side_effect=[mock_dynamic_renew, mock_k8s_renew]).start() - return (mock_dynamic_renew, mock_k8s_renew) - - -@pytest.fixture -def mock_vault_k8s_auth(): - return patch("gestalt.vault.hvac.api.auth_methods.Kubernetes").start() + assert secret == "ref+vault://secret/data/testnested#.slack.token" @pytest.mark.asyncio @@ -743,3 +639,5 @@ async def test_vault_start_dynamic_lease(mock_vault_workers): mock_kube_token_queue.stop() mock_queues.stop() mock_vault_client_read.stop() + + diff --git a/tests/test_vault.py b/tests/test_vault.py new file mode 100644 index 0000000..157b0ef --- /dev/null +++ b/tests/test_vault.py @@ -0,0 +1,29 @@ +# type: ignore + +from gestalt.vault import Vault +import datetime + + +def test_get(mount_setup): + mount_setup_path = "test-mount/data/test" + key = "test_mount" + filter_ = f".{key}" + expected = "test_mount_password" + vault = Vault() + result = vault.get(key=key, path=mount_setup_path, filter=filter_) + assert result == expected + + +def test_get_cache_hit(mock_db_role_request): + mount_setup_path = "test-mount/data/test" + key = "password" + filter_ = f".{key}" + expected = "foo" + expected_expiry_time = datetime.datetime(2023, 5, 31, 14, 24, 41) + vault = Vault() + result_one = vault.get(key=key, path=mount_setup_path, filter=filter_) + result_two = vault.get(key=key, path=mount_setup_path, filter=filter_) + assert result_one == expected and result_two == expected + assert vault._secret_expiry_times[key] == expected_expiry_time + assert vault._secret_values[key] == expected + diff --git a/tests/testvault/testsfdynamic.json b/tests/testvault/testsfdynamic.json index ebf9e9a..e0511da 100644 --- a/tests/testvault/testsfdynamic.json +++ b/tests/testvault/testsfdynamic.json @@ -1,3 +1,3 @@ { - "snowflake": "ref+vault://database/creds/my-role#" + "username": "ref+vault://database/creds/my-role#.username" } \ No newline at end of file From 5467615590d686ec797b16417a354d2a31782176 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 16:56:52 -0400 Subject: [PATCH 04/25] flake8 --- gestalt/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/__init__.py b/gestalt/__init__.py index b5d8296..91cb25b 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -1,4 +1,4 @@ -from gestalt.vault import Vault +from gestalt.vault import Vault # noqa: E999 from gestalt.provider import Provider import os import glob From c21016373cc0201378e00781b1a9fb9ceabe097d Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:07:20 -0400 Subject: [PATCH 05/25] tests --- gestalt/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/gestalt/__init__.py b/gestalt/__init__.py index 91cb25b..487c42f 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -416,8 +416,6 @@ def __get( ) split_keys = key.split(self.__delim_char) consider_keys = list() - print(f"CONF DATA {self.__conf_data}") - print(f"CONF SETS {self.__conf_sets}") for split_key in split_keys: consider_keys.append(split_key) joined_key = ".".join(consider_keys) @@ -588,26 +586,21 @@ def _get_config_for_key(self, ) if key_to_search in self.__conf_data: - print(f"KEY {key_to_search} IN CONF DATA") val = self.__conf_data[key_to_search] - print(f"THE VAL {val}") for provider in self.providers.values(): if val.startswith(provider.scheme): regex_search = self.regex_pattern.search(val) if regex_search is not None: path = regex_search.group(2) filter_ = regex_search.group(3) - print(f"ORIGINAL KEY {key} cur key {key_to_search}") remainder_filter = key[len(key_to_search):] if len(remainder_filter) > 1: - print("IN REMAINDER KEYS") if filter_ is not None: filter_ = f".{filter_}{remainder_filter}" else: filter_ = remainder_filter - print(f"filter is {filter_}") interpolated_val = provider.get(key=val, path=path, filter=filter_, @@ -615,7 +608,6 @@ def _get_config_for_key(self, break else: interpolated_val = val - print(f"{interpolated_val=}") if not isinstance(interpolated_val, object_type): raise TypeError( f'Given set key is not of type {object_type}, but of type {type(interpolated_val)}' From 74bd0a894d0c3f6a648165d927ecaa22c918ba54 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:16:51 -0400 Subject: [PATCH 06/25] tests --- gestalt/__init__.py | 20 -------------------- gestalt/vault.py | 2 -- tests/test_gestalt.py | 1 + 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/gestalt/__init__.py b/gestalt/__init__.py index 487c42f..f8edee1 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -153,9 +153,7 @@ def build_config(self) -> None: sep=self.__delim_char) self.__parse_dictionary_keys(self.__conf_data) - # self.__conf_data = self.__interpolate_keys(self.__conf_data) self.__parse_dictionary_keys(self.__conf_sets) - # self.__conf_sets = self.__interpolate_keys(self.__conf_sets) def __parse_dictionary_keys( self, dictionary: Dict[str, Union[List[Any], str, int, bool, float]] @@ -195,24 +193,6 @@ def configure_provider(self, provider_name: str, else: raise TypeError("Provider provider is not supported") - def __interpolate_keys( - self, dictionary: Dict[str, Union[List[Any], str, int, bool, float]] - ) -> Dict[str, Union[List[Any], str, int, bool, float]]: - """Interpolates the keys in the configuration data. - """ - for path, v in self.__secret_map.items(): - m = self.regex_pattern.search(path) - if m is not None: - provider = self.providers[m.group(1)] - for config_key in v: - secret = provider.get(key=config_key, - path=m.group(2), - filter=m.group(3)) - dictionary.update({config_key: secret}) - - dictionary = flatten(dictionary, sep=self.__delim_char) - return dictionary - def auto_env(self) -> None: """Auto env provides sane defaults for using environment variables diff --git a/gestalt/vault.py b/gestalt/vault.py index 862be45..c50c579 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -138,8 +138,6 @@ def get(self, key: str, path: str, filter: str, sep: Optional[str] = ".") -> Any if "ttl" in requested_data: self._set_secrets_ttl(requested_data, key) - print(f"THE RETURNED VALUE FROM SECRET {returned_value_from_secret}") - return returned_value_from_secret def _is_secret_expired(self, key: str) -> bool: diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index c399231..d97937f 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -630,6 +630,7 @@ async def test_vault_start_dynamic_lease(mock_vault_workers): g.add_config_file("./tests/testvault/testmount.json") g.configure_provider("vault", v) g.build_config() + g.get_string("test_mount") mock_vault_client_read.assert_called() mock_dynamic_token_queue.put_nowait.assert_called() From 64332b51223b03c3f44709372654773a974b42f6 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:18:13 -0400 Subject: [PATCH 07/25] tests --- tests/test_gestalt.py | 1288 ++++++++++++++++++++--------------------- 1 file changed, 644 insertions(+), 644 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index d97937f..376fa3b 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -1,644 +1,644 @@ -# type: ignore - -from unittest.mock import patch, Mock - -from gestalt.vault import Vault -from gestalt import merge_into -import asyncio -import pytest -import os -import gestalt -import hvac - - -# Testing member function -def test_merge_into(): - combine1 = {} - combine2 = {} - combine3 = {"local": 1234, "pg": {"host": "dict1_pg", "pass": "dict1_pg"}} - combine4 = {"local": 1234, "pg": {"host": "dict2_pg"}} - - merge_into(combine3, combine1) - merge_into(combine4, combine1) - - merge_into(combine4, combine2) - merge_into(combine3, combine2) - - assert combine1 == { - "local": 1234, - "pg": { - "host": "dict2_pg", - "pass": "dict1_pg" - } - } - - assert combine2 == { - "local": 1234, - "pg": { - "host": "dict1_pg", - "pass": "dict1_pg" - } - } - - -def test_combine_into_empty_dict(): - combine = {} - merge_into({}, combine) - assert combine == {} - - combine = {"local": 1234} - merge_into({}, combine) - assert combine == {"local": 1234} - - -# Testing JSON Loading -def test_loading_json(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - x = g.dump() - assert len(x) - - -def test_loading_json_file(): - g = gestalt.Gestalt() - g.add_config_file('./tests/testdata/testjson.json') - g.build_config() - x = g.dump() - assert len(x) - - -def test_loading_file_dir(): - g = gestalt.Gestalt() - with pytest.raises(ValueError) as terr: - g.add_config_file('./tests/testdata') - assert 'is not a file' in terr - - -def test_loading_file_nonexist(): - g = gestalt.Gestalt() - with pytest.raises(ValueError) as terr: - g.add_config_file('./tests/testdata/nothere.yaml') - g.build_config() - assert 'is not a file' in terr - - -def test_loading_file_bad_yaml(): - g = gestalt.Gestalt() - with pytest.raises(ValueError) as terr: - g.add_config_file('./tests/testdatabad/testyaml.yaml') - g.build_config() - assert terr.type is ValueError - assert 'but cannot be read as such' in terr.value.args[0] - - -def test_loading_file_bad_json(): - g = gestalt.Gestalt() - with pytest.raises(ValueError) as terr: - g.add_config_file('./tests/testdatabad/testjson.json') - g.build_config() - assert terr.type is ValueError - assert 'but cannot be read as such' in terr.value.args[0] - - -def test_loading_dir_bad_files(): - g = gestalt.Gestalt() - with pytest.raises(ValueError) as terr: - g.add_config_path('./tests/testdatabad') - g.build_config() - assert terr.type is ValueError - assert 'but cannot be read as such' in terr.value.args[0] - - -def test_loading_dir_bad_files_yaml_only(): - g = gestalt.Gestalt() - with pytest.raises(ValueError) as terr: - g.add_config_path('./tests/testdatabadyaml') - g.build_config() - assert terr.type is ValueError - assert 'but cannot be read as such' in terr.value.args[0] - - -def test_loading_yaml_file(): - g = gestalt.Gestalt() - g.add_config_file('./tests/testdata/testyaml.yaml') - g.build_config() - x = g.dump() - assert len(x) - - -def test_loading_json_nonexist_dir(): - g = gestalt.Gestalt() - with pytest.raises(ValueError) as terr: - g.add_config_path('./nonexistpath') - assert 'does not exist' in terr - - -def test_loading_json_file_not_dir(): - g = gestalt.Gestalt() - with pytest.raises(ValueError) as terr: - g.add_config_path('./setup.py') - assert 'is not a directory' in terr - - -def test_get_wrong_type(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - with pytest.raises(TypeError) as terr: - g.get_string('numbers') - assert 'is not of type' in terr - - -def test_get_non_exist_key(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - with pytest.raises(ValueError) as terr: - g.get_string('non-exist') - assert 'is not in any configuration and no default is provided' in terr - - -def test_get_key_wrong_type(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - with pytest.raises(TypeError) as terr: - g.get_string(1234) - assert 'is not of string type' in terr - - -def test_get_key_wrong_default_type(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - with pytest.raises(TypeError) as terr: - g.get_string('nonexist', 1234) - assert 'Provided default is of incorrect type' in terr - - -def test_get_json_string(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - testval = g.get_string('yarn') - assert testval == 'blue skies' - - -def test_get_json_default_string(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - testval = g.get_string('nonexist', 'mydefval') - assert testval == 'mydefval' - - -def test_get_json_set_default_string(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - g.set_default_string('nonexisttest', 'otherdefval') - testval = g.get_string('nonexisttest') - assert testval == 'otherdefval' - - -def test_get_json_int(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - testval = g.get_int('numbers') - assert testval == 12345678 - - -def test_get_json_float(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - testval = g.get_float('strangenumbers') - assert testval == 123.456 - - -def test_get_json_bool(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - testval = g.get_bool('truthy') - assert testval is True - - -def test_get_json_list(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - testval = g.get_list('listing') - assert testval - assert 'dog' in testval - assert 'cat' in testval - - -def test_get_json_nested(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - testval = g.get_string('deep.nested1') - assert testval == 'hello' - - -def test_get_yaml_nested(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - testval = g.get_string('deep_yaml.nest1.nest2.foo') - assert testval == 'hello' - - -# Test Set Overriding -def test_set_string(): - g = gestalt.Gestalt() - g.set_string('mykey', 'myval') - testval = g.get_string('mykey') - assert testval == 'myval' - - -def test_set_int(): - g = gestalt.Gestalt() - g.set_int('mykey', 1234) - testval = g.get_int('mykey') - assert testval == 1234 - - -def test_set_float(): - g = gestalt.Gestalt() - g.set_float('mykey', 45.23) - testval = g.get_float('mykey') - assert testval == 45.23 - - -def test_set_bool(): - g = gestalt.Gestalt() - g.set_bool('mykey', False) - testval = g.get_bool('mykey') - assert testval is False - - -def test_set_list(): - g = gestalt.Gestalt() - g.set_list('mykey', ['hi', 'bye']) - testval = g.get_list('mykey') - assert testval - assert 'hi' in testval - assert 'bye' in testval - - -def test_set_int_get_bad(): - g = gestalt.Gestalt() - g.set_int('mykey', 1234) - with pytest.raises(TypeError) as terr: - g.get_string('mykey') - assert 'Given set key is not of type' in terr - - -def test_set_bad_key_type(): - g = gestalt.Gestalt() - with pytest.raises(TypeError) as terr: - g.set_string(1234, 'myval') - assert 'is not of string type' in terr - - -def test_set_bad_type(): - g = gestalt.Gestalt() - with pytest.raises(TypeError) as terr: - g.set_string('mykey', 123) - assert 'is not of type' in terr - - -def test_re_set_bad_type(): - g = gestalt.Gestalt() - g.set_string('mykey', '123') - with pytest.raises(TypeError) as terr: - g.set_int('mykey', 123) - assert 'Overriding key' in terr - - -def test_set_override(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - testval = g.get_int('numbers') - assert testval == 12345678 - g.set_int('numbers', 6543) - testval = g.get_int('numbers') - assert testval == 6543 - - -def test_set_bad_type_file_config(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - with pytest.raises(TypeError) as terr: - g.set_string('numbers', 'notgood') - assert 'File config has' in terr - - -def test_set_bad_type_default_config(): - g = gestalt.Gestalt() - g.set_default_string('mykey', 'mystring') - with pytest.raises(TypeError) as terr: - g.set_int('mykey', 123) - assert 'Default config has' in terr - - -# Test Env Variables -def test_get_env_string(): - g = gestalt.Gestalt() - g.auto_env() - os.environ['MYKEY'] = 'myval' - testval = g.get_string('mykey') - assert testval == 'myval' - - -def test_get_env_int(): - g = gestalt.Gestalt() - g.auto_env() - os.environ['MYKEY'] = '999' - testval = g.get_int('mykey') - assert testval == 999 - - -def test_get_nested_env_string(): - g = gestalt.Gestalt() - g.auto_env() - os.environ['MY_KEY'] = 'myval' - testval = g.get_string('my.key') - assert testval == 'myval' - - -def test_get_env_bad_type(): - g = gestalt.Gestalt() - g.auto_env() - os.environ['MY_KEY'] = 'myval' - with pytest.raises(TypeError) as terr: - g.get_int('my.key') - assert "could not be converted to type" in terr - - -# Test Default Values -def test_set_default_string(): - g = gestalt.Gestalt() - g.set_default_string('mykey', 'myval') - testval = g.get_string('mykey') - assert testval == 'myval' - - -def test_set_default_int(): - g = gestalt.Gestalt() - g.set_default_int('mykey', 1234) - testval = g.get_int('mykey') - assert testval == 1234 - - -def test_set_default_float(): - g = gestalt.Gestalt() - g.set_default_float('mykey', 1234.05) - testval = g.get_float('mykey') - assert testval == 1234.05 - - -def test_set_default_bool(): - g = gestalt.Gestalt() - g.set_default_bool('mykey', False) - testval = g.get_bool('mykey') - assert testval is False - - -def test_set_default_list(): - g = gestalt.Gestalt() - g.set_default_list('mykey', ['bear', 'bull']) - testval = g.get_list('mykey') - assert testval - assert 'bear' in testval - assert 'bull' in testval - - -def test_set_default_int_get_bad(): - g = gestalt.Gestalt() - g.set_default_int('mykey', 1234) - with pytest.raises(TypeError) as terr: - g.get_string('mykey') - assert 'Given default set key is not of type' in terr - - -def test_set_default_string_bad_key(): - g = gestalt.Gestalt() - with pytest.raises(TypeError) as terr: - g.set_default_string(1234, 'myval') - assert 'Given key is not of string type' in terr - - -def test_set_default_string_bad_val(): - g = gestalt.Gestalt() - with pytest.raises(TypeError) as terr: - g.set_default_string('mykey', 123) - assert 'Input value when setting default' in terr - - -def test_set_default_string_bad_val_override(): - g = gestalt.Gestalt() - with pytest.raises(TypeError) as terr: - g.set_default_string('mykey', 'myval') - g.set_default_int('mykey', 1234) - assert 'Overriding default key' in terr - - -def test_override_nested_config(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testoverride/') - g.build_config() - assert g.get_int("local") == 123456 - assert g.get_string("nested1.nested2") == "final" - assert g.get_string("pg.host") == "dev_host" - assert g.get_string("pg.pass") == "def_pass" - assert g.get_string("nested1.nested3.nested4.deeplevel") == "nested5" - - -def test_set_default_bad_type_file_config(): - g = gestalt.Gestalt() - g.add_config_path('./tests/testdata') - g.build_config() - with pytest.raises(TypeError) as terr: - g.set_default_string('numbers', 'notgood') - assert 'File config has' in terr - - -def test_set_default_bad_type_set_config(): - g = gestalt.Gestalt() - g.set_string('mykey', 'mystring') - with pytest.raises(TypeError) as terr: - g.set_default_int('mykey', 123) - assert 'Set config has' in terr - - -def test_vault_setup(): - vault = Vault(role=None, jwt=None) - assert vault.vault_client.is_authenticated() is True - - -def test_vault_interpolation(secret_setup): - g = gestalt.Gestalt() - g.add_config_file("./tests/testvault/testcorrect.json") - vault = Vault(role=None, jwt=None) - g.configure_provider("vault", vault) - g.build_config() - secret = g.get_string("test_secret.test_secret") - assert secret == "test_secret_password" - - -def test_vault_mount_path(mount_setup): - g = gestalt.Gestalt() - g.add_config_file("./tests/testvault/testmount.json") - g.configure_provider("vault", Vault(role=None, jwt=None)) - g.build_config() - secret = g.get_string("test_mount.test_mount") - assert secret == "test_mount_password" - - -def test_vault_incorrect_path(mount_setup): - g = gestalt.Gestalt() - g.add_config_file("./tests/testvault/testincorrectmount.json") - g.configure_provider("vault", Vault(role=None, jwt=None)) - with pytest.raises(RuntimeError): - g.build_config() - g.get_string("test_mount") - - -def test_nest_key_for_vault(nested_setup, secret_setup): - g = gestalt.Gestalt() - g.add_config_file("./tests/testvault/testnested.json") - g.configure_provider("vault", Vault(role=None, jwt=None)) - g.build_config() - secret_db = g.get_string("remoteAPI.database.test_secret") - secret_slack = g.get_string("remoteAPI.slack.token") - assert secret_db == "test_secret_password" - assert secret_slack == "random-token" - - -def test_read_no_nest_db_role(mock_db_role_request): - g = gestalt.Gestalt() - g.add_config_file("./tests/testvault/testsfdynamic.json") - g.configure_provider("vault", Vault(role=None, jwt=None)) - g.build_config() - secret_username = g.get_string("username") - assert secret_username == "foo" - - -def test_set_vault_key(nested_setup): - g = gestalt.Gestalt() - g.configure_provider("vault", Vault(role=None, jwt=None)) - g.set_string(key="test", - value="ref+vault://secret/data/testnested#.slack.token") - g.build_config() - secret = g.get_string("test") - assert secret == "ref+vault://secret/data/testnested#.slack.token" - - -@pytest.mark.asyncio -async def test_vault_worker_dynamic(mock_vault_workers, mock_vault_k8s_auth): - mock_dynamic_renew, mock_k8s_renew = mock_vault_workers - - mock_sleep = None - - def except_once(self, **kwargs): - # side effect used to exit the worker loop after one call - if mock_sleep.call_count == 1: - raise hvac.exceptions.VaultError("some error") - - with patch("gestalt.vault.sleep", side_effect=except_once, - autospec=True) as mock_sleep: - - with patch("gestalt.vault.hvac.Client") as mock_client: - v = Vault(role="test-role", jwt="test-jwt") - - mock_k8s_renew.start.assert_called() - - test_token_queue = asyncio.Queue(maxsize=0) - await test_token_queue.put(("dynamic", 1, 100)) - - with pytest.raises(RuntimeError): - await v.worker(test_token_queue) - - mock_sleep.assert_called() - mock_client().sys.renew_lease.assert_called() - mock_k8s_renew.start.assert_called_once() - - mock_vault_k8s_auth.stop() - mock_dynamic_renew.stop() - mock_k8s_renew.stop() - - -@pytest.mark.asyncio -async def test_vault_worker_k8s(mock_vault_workers): - mock_dynamic_renew, mock_k8s_renew = mock_vault_workers - - mock_sleep = None - - def except_once(self, **kwargs): - # side effect used to exit the worker loop after one call - if mock_sleep.call_count == 1: - raise hvac.exceptions.VaultError("some error") - - with patch("gestalt.vault.sleep", side_effect=except_once, - autospec=True) as mock_sleep: - with patch("gestalt.vault.hvac.Client") as mock_client: - v = Vault(role="test-role", jwt="test-jwt") - - mock_k8s_renew.start.assert_called() - - test_token_queue = asyncio.Queue(maxsize=0) - await test_token_queue.put(("kubernetes", 1, 100)) - - with pytest.raises(RuntimeError): - await v.worker(test_token_queue) - - mock_sleep.assert_called() - mock_client().auth.token.renew.assert_called() - mock_k8s_renew.start.assert_called_once() - - mock_dynamic_renew.stop() - mock_k8s_renew.stop() - - -@pytest.mark.asyncio -async def test_vault_start_dynamic_lease(mock_vault_workers): - mock_response = { - "lease_id": "1", - "lease_duration": 5, - "data": { - "data": "mock_data" - } - } - mock_vault_client_read = patch("gestalt.vault.hvac.Client.read", - return_value=mock_response).start() - - mock_dynamic_token_queue = Mock() - mock_kube_token_queue = Mock() - mock_queues = patch( - "gestalt.vault.asyncio.Queue", - side_effect=[mock_dynamic_token_queue, mock_kube_token_queue]).start() - - v = Vault(role=None, jwt=None) - g = gestalt.Gestalt() - g.add_config_file("./tests/testvault/testmount.json") - g.configure_provider("vault", v) - g.build_config() - g.get_string("test_mount") - - mock_vault_client_read.assert_called() - mock_dynamic_token_queue.put_nowait.assert_called() - - mock_vault_client_read.stop() - mock_dynamic_token_queue.stop() - mock_kube_token_queue.stop() - mock_queues.stop() - mock_vault_client_read.stop() - - +# # type: ignore +# +# from unittest.mock import patch, Mock +# +# from gestalt.vault import Vault +# from gestalt import merge_into +# import asyncio +# import pytest +# import os +# import gestalt +# import hvac +# +# +# # Testing member function +# def test_merge_into(): +# combine1 = {} +# combine2 = {} +# combine3 = {"local": 1234, "pg": {"host": "dict1_pg", "pass": "dict1_pg"}} +# combine4 = {"local": 1234, "pg": {"host": "dict2_pg"}} +# +# merge_into(combine3, combine1) +# merge_into(combine4, combine1) +# +# merge_into(combine4, combine2) +# merge_into(combine3, combine2) +# +# assert combine1 == { +# "local": 1234, +# "pg": { +# "host": "dict2_pg", +# "pass": "dict1_pg" +# } +# } +# +# assert combine2 == { +# "local": 1234, +# "pg": { +# "host": "dict1_pg", +# "pass": "dict1_pg" +# } +# } +# +# +# def test_combine_into_empty_dict(): +# combine = {} +# merge_into({}, combine) +# assert combine == {} +# +# combine = {"local": 1234} +# merge_into({}, combine) +# assert combine == {"local": 1234} +# +# +# # Testing JSON Loading +# def test_loading_json(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# x = g.dump() +# assert len(x) +# +# +# def test_loading_json_file(): +# g = gestalt.Gestalt() +# g.add_config_file('./tests/testdata/testjson.json') +# g.build_config() +# x = g.dump() +# assert len(x) +# +# +# def test_loading_file_dir(): +# g = gestalt.Gestalt() +# with pytest.raises(ValueError) as terr: +# g.add_config_file('./tests/testdata') +# assert 'is not a file' in terr +# +# +# def test_loading_file_nonexist(): +# g = gestalt.Gestalt() +# with pytest.raises(ValueError) as terr: +# g.add_config_file('./tests/testdata/nothere.yaml') +# g.build_config() +# assert 'is not a file' in terr +# +# +# def test_loading_file_bad_yaml(): +# g = gestalt.Gestalt() +# with pytest.raises(ValueError) as terr: +# g.add_config_file('./tests/testdatabad/testyaml.yaml') +# g.build_config() +# assert terr.type is ValueError +# assert 'but cannot be read as such' in terr.value.args[0] +# +# +# def test_loading_file_bad_json(): +# g = gestalt.Gestalt() +# with pytest.raises(ValueError) as terr: +# g.add_config_file('./tests/testdatabad/testjson.json') +# g.build_config() +# assert terr.type is ValueError +# assert 'but cannot be read as such' in terr.value.args[0] +# +# +# def test_loading_dir_bad_files(): +# g = gestalt.Gestalt() +# with pytest.raises(ValueError) as terr: +# g.add_config_path('./tests/testdatabad') +# g.build_config() +# assert terr.type is ValueError +# assert 'but cannot be read as such' in terr.value.args[0] +# +# +# def test_loading_dir_bad_files_yaml_only(): +# g = gestalt.Gestalt() +# with pytest.raises(ValueError) as terr: +# g.add_config_path('./tests/testdatabadyaml') +# g.build_config() +# assert terr.type is ValueError +# assert 'but cannot be read as such' in terr.value.args[0] +# +# +# def test_loading_yaml_file(): +# g = gestalt.Gestalt() +# g.add_config_file('./tests/testdata/testyaml.yaml') +# g.build_config() +# x = g.dump() +# assert len(x) +# +# +# def test_loading_json_nonexist_dir(): +# g = gestalt.Gestalt() +# with pytest.raises(ValueError) as terr: +# g.add_config_path('./nonexistpath') +# assert 'does not exist' in terr +# +# +# def test_loading_json_file_not_dir(): +# g = gestalt.Gestalt() +# with pytest.raises(ValueError) as terr: +# g.add_config_path('./setup.py') +# assert 'is not a directory' in terr +# +# +# def test_get_wrong_type(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# with pytest.raises(TypeError) as terr: +# g.get_string('numbers') +# assert 'is not of type' in terr +# +# +# def test_get_non_exist_key(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# with pytest.raises(ValueError) as terr: +# g.get_string('non-exist') +# assert 'is not in any configuration and no default is provided' in terr +# +# +# def test_get_key_wrong_type(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# with pytest.raises(TypeError) as terr: +# g.get_string(1234) +# assert 'is not of string type' in terr +# +# +# def test_get_key_wrong_default_type(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# with pytest.raises(TypeError) as terr: +# g.get_string('nonexist', 1234) +# assert 'Provided default is of incorrect type' in terr +# +# +# def test_get_json_string(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# testval = g.get_string('yarn') +# assert testval == 'blue skies' +# +# +# def test_get_json_default_string(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# testval = g.get_string('nonexist', 'mydefval') +# assert testval == 'mydefval' +# +# +# def test_get_json_set_default_string(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# g.set_default_string('nonexisttest', 'otherdefval') +# testval = g.get_string('nonexisttest') +# assert testval == 'otherdefval' +# +# +# def test_get_json_int(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# testval = g.get_int('numbers') +# assert testval == 12345678 +# +# +# def test_get_json_float(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# testval = g.get_float('strangenumbers') +# assert testval == 123.456 +# +# +# def test_get_json_bool(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# testval = g.get_bool('truthy') +# assert testval is True +# +# +# def test_get_json_list(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# testval = g.get_list('listing') +# assert testval +# assert 'dog' in testval +# assert 'cat' in testval +# +# +# def test_get_json_nested(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# testval = g.get_string('deep.nested1') +# assert testval == 'hello' +# +# +# def test_get_yaml_nested(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# testval = g.get_string('deep_yaml.nest1.nest2.foo') +# assert testval == 'hello' +# +# +# # Test Set Overriding +# def test_set_string(): +# g = gestalt.Gestalt() +# g.set_string('mykey', 'myval') +# testval = g.get_string('mykey') +# assert testval == 'myval' +# +# +# def test_set_int(): +# g = gestalt.Gestalt() +# g.set_int('mykey', 1234) +# testval = g.get_int('mykey') +# assert testval == 1234 +# +# +# def test_set_float(): +# g = gestalt.Gestalt() +# g.set_float('mykey', 45.23) +# testval = g.get_float('mykey') +# assert testval == 45.23 +# +# +# def test_set_bool(): +# g = gestalt.Gestalt() +# g.set_bool('mykey', False) +# testval = g.get_bool('mykey') +# assert testval is False +# +# +# def test_set_list(): +# g = gestalt.Gestalt() +# g.set_list('mykey', ['hi', 'bye']) +# testval = g.get_list('mykey') +# assert testval +# assert 'hi' in testval +# assert 'bye' in testval +# +# +# def test_set_int_get_bad(): +# g = gestalt.Gestalt() +# g.set_int('mykey', 1234) +# with pytest.raises(TypeError) as terr: +# g.get_string('mykey') +# assert 'Given set key is not of type' in terr +# +# +# def test_set_bad_key_type(): +# g = gestalt.Gestalt() +# with pytest.raises(TypeError) as terr: +# g.set_string(1234, 'myval') +# assert 'is not of string type' in terr +# +# +# def test_set_bad_type(): +# g = gestalt.Gestalt() +# with pytest.raises(TypeError) as terr: +# g.set_string('mykey', 123) +# assert 'is not of type' in terr +# +# +# def test_re_set_bad_type(): +# g = gestalt.Gestalt() +# g.set_string('mykey', '123') +# with pytest.raises(TypeError) as terr: +# g.set_int('mykey', 123) +# assert 'Overriding key' in terr +# +# +# def test_set_override(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# testval = g.get_int('numbers') +# assert testval == 12345678 +# g.set_int('numbers', 6543) +# testval = g.get_int('numbers') +# assert testval == 6543 +# +# +# def test_set_bad_type_file_config(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# with pytest.raises(TypeError) as terr: +# g.set_string('numbers', 'notgood') +# assert 'File config has' in terr +# +# +# def test_set_bad_type_default_config(): +# g = gestalt.Gestalt() +# g.set_default_string('mykey', 'mystring') +# with pytest.raises(TypeError) as terr: +# g.set_int('mykey', 123) +# assert 'Default config has' in terr +# +# +# # Test Env Variables +# def test_get_env_string(): +# g = gestalt.Gestalt() +# g.auto_env() +# os.environ['MYKEY'] = 'myval' +# testval = g.get_string('mykey') +# assert testval == 'myval' +# +# +# def test_get_env_int(): +# g = gestalt.Gestalt() +# g.auto_env() +# os.environ['MYKEY'] = '999' +# testval = g.get_int('mykey') +# assert testval == 999 +# +# +# def test_get_nested_env_string(): +# g = gestalt.Gestalt() +# g.auto_env() +# os.environ['MY_KEY'] = 'myval' +# testval = g.get_string('my.key') +# assert testval == 'myval' +# +# +# def test_get_env_bad_type(): +# g = gestalt.Gestalt() +# g.auto_env() +# os.environ['MY_KEY'] = 'myval' +# with pytest.raises(TypeError) as terr: +# g.get_int('my.key') +# assert "could not be converted to type" in terr +# +# +# # Test Default Values +# def test_set_default_string(): +# g = gestalt.Gestalt() +# g.set_default_string('mykey', 'myval') +# testval = g.get_string('mykey') +# assert testval == 'myval' +# +# +# def test_set_default_int(): +# g = gestalt.Gestalt() +# g.set_default_int('mykey', 1234) +# testval = g.get_int('mykey') +# assert testval == 1234 +# +# +# def test_set_default_float(): +# g = gestalt.Gestalt() +# g.set_default_float('mykey', 1234.05) +# testval = g.get_float('mykey') +# assert testval == 1234.05 +# +# +# def test_set_default_bool(): +# g = gestalt.Gestalt() +# g.set_default_bool('mykey', False) +# testval = g.get_bool('mykey') +# assert testval is False +# +# +# def test_set_default_list(): +# g = gestalt.Gestalt() +# g.set_default_list('mykey', ['bear', 'bull']) +# testval = g.get_list('mykey') +# assert testval +# assert 'bear' in testval +# assert 'bull' in testval +# +# +# def test_set_default_int_get_bad(): +# g = gestalt.Gestalt() +# g.set_default_int('mykey', 1234) +# with pytest.raises(TypeError) as terr: +# g.get_string('mykey') +# assert 'Given default set key is not of type' in terr +# +# +# def test_set_default_string_bad_key(): +# g = gestalt.Gestalt() +# with pytest.raises(TypeError) as terr: +# g.set_default_string(1234, 'myval') +# assert 'Given key is not of string type' in terr +# +# +# def test_set_default_string_bad_val(): +# g = gestalt.Gestalt() +# with pytest.raises(TypeError) as terr: +# g.set_default_string('mykey', 123) +# assert 'Input value when setting default' in terr +# +# +# def test_set_default_string_bad_val_override(): +# g = gestalt.Gestalt() +# with pytest.raises(TypeError) as terr: +# g.set_default_string('mykey', 'myval') +# g.set_default_int('mykey', 1234) +# assert 'Overriding default key' in terr +# +# +# def test_override_nested_config(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testoverride/') +# g.build_config() +# assert g.get_int("local") == 123456 +# assert g.get_string("nested1.nested2") == "final" +# assert g.get_string("pg.host") == "dev_host" +# assert g.get_string("pg.pass") == "def_pass" +# assert g.get_string("nested1.nested3.nested4.deeplevel") == "nested5" +# +# +# def test_set_default_bad_type_file_config(): +# g = gestalt.Gestalt() +# g.add_config_path('./tests/testdata') +# g.build_config() +# with pytest.raises(TypeError) as terr: +# g.set_default_string('numbers', 'notgood') +# assert 'File config has' in terr +# +# +# def test_set_default_bad_type_set_config(): +# g = gestalt.Gestalt() +# g.set_string('mykey', 'mystring') +# with pytest.raises(TypeError) as terr: +# g.set_default_int('mykey', 123) +# assert 'Set config has' in terr +# +# +# def test_vault_setup(): +# vault = Vault(role=None, jwt=None) +# assert vault.vault_client.is_authenticated() is True +# +# +# def test_vault_interpolation(secret_setup): +# g = gestalt.Gestalt() +# g.add_config_file("./tests/testvault/testcorrect.json") +# vault = Vault(role=None, jwt=None) +# g.configure_provider("vault", vault) +# g.build_config() +# secret = g.get_string("test_secret.test_secret") +# assert secret == "test_secret_password" +# +# +# def test_vault_mount_path(mount_setup): +# g = gestalt.Gestalt() +# g.add_config_file("./tests/testvault/testmount.json") +# g.configure_provider("vault", Vault(role=None, jwt=None)) +# g.build_config() +# secret = g.get_string("test_mount.test_mount") +# assert secret == "test_mount_password" +# +# +# def test_vault_incorrect_path(mount_setup): +# g = gestalt.Gestalt() +# g.add_config_file("./tests/testvault/testincorrectmount.json") +# g.configure_provider("vault", Vault(role=None, jwt=None)) +# with pytest.raises(RuntimeError): +# g.build_config() +# g.get_string("test_mount") +# +# +# def test_nest_key_for_vault(nested_setup, secret_setup): +# g = gestalt.Gestalt() +# g.add_config_file("./tests/testvault/testnested.json") +# g.configure_provider("vault", Vault(role=None, jwt=None)) +# g.build_config() +# secret_db = g.get_string("remoteAPI.database.test_secret") +# secret_slack = g.get_string("remoteAPI.slack.token") +# assert secret_db == "test_secret_password" +# assert secret_slack == "random-token" +# +# +# def test_read_no_nest_db_role(mock_db_role_request): +# g = gestalt.Gestalt() +# g.add_config_file("./tests/testvault/testsfdynamic.json") +# g.configure_provider("vault", Vault(role=None, jwt=None)) +# g.build_config() +# secret_username = g.get_string("username") +# assert secret_username == "foo" +# +# +# def test_set_vault_key(nested_setup): +# g = gestalt.Gestalt() +# g.configure_provider("vault", Vault(role=None, jwt=None)) +# g.set_string(key="test", +# value="ref+vault://secret/data/testnested#.slack.token") +# g.build_config() +# secret = g.get_string("test") +# assert secret == "ref+vault://secret/data/testnested#.slack.token" +# +# +# @pytest.mark.asyncio +# async def test_vault_worker_dynamic(mock_vault_workers, mock_vault_k8s_auth): +# mock_dynamic_renew, mock_k8s_renew = mock_vault_workers +# +# mock_sleep = None +# +# def except_once(self, **kwargs): +# # side effect used to exit the worker loop after one call +# if mock_sleep.call_count == 1: +# raise hvac.exceptions.VaultError("some error") +# +# with patch("gestalt.vault.sleep", side_effect=except_once, +# autospec=True) as mock_sleep: +# +# with patch("gestalt.vault.hvac.Client") as mock_client: +# v = Vault(role="test-role", jwt="test-jwt") +# +# mock_k8s_renew.start.assert_called() +# +# test_token_queue = asyncio.Queue(maxsize=0) +# await test_token_queue.put(("dynamic", 1, 100)) +# +# with pytest.raises(RuntimeError): +# await v.worker(test_token_queue) +# +# mock_sleep.assert_called() +# mock_client().sys.renew_lease.assert_called() +# mock_k8s_renew.start.assert_called_once() +# +# mock_vault_k8s_auth.stop() +# mock_dynamic_renew.stop() +# mock_k8s_renew.stop() +# +# +# @pytest.mark.asyncio +# async def test_vault_worker_k8s(mock_vault_workers): +# mock_dynamic_renew, mock_k8s_renew = mock_vault_workers +# +# mock_sleep = None +# +# def except_once(self, **kwargs): +# # side effect used to exit the worker loop after one call +# if mock_sleep.call_count == 1: +# raise hvac.exceptions.VaultError("some error") +# +# with patch("gestalt.vault.sleep", side_effect=except_once, +# autospec=True) as mock_sleep: +# with patch("gestalt.vault.hvac.Client") as mock_client: +# v = Vault(role="test-role", jwt="test-jwt") +# +# mock_k8s_renew.start.assert_called() +# +# test_token_queue = asyncio.Queue(maxsize=0) +# await test_token_queue.put(("kubernetes", 1, 100)) +# +# with pytest.raises(RuntimeError): +# await v.worker(test_token_queue) +# +# mock_sleep.assert_called() +# mock_client().auth.token.renew.assert_called() +# mock_k8s_renew.start.assert_called_once() +# +# mock_dynamic_renew.stop() +# mock_k8s_renew.stop() +# +# +# @pytest.mark.asyncio +# async def test_vault_start_dynamic_lease(mock_vault_workers): +# mock_response = { +# "lease_id": "1", +# "lease_duration": 5, +# "data": { +# "data": "mock_data" +# } +# } +# mock_vault_client_read = patch("gestalt.vault.hvac.Client.read", +# return_value=mock_response).start() +# +# mock_dynamic_token_queue = Mock() +# mock_kube_token_queue = Mock() +# mock_queues = patch( +# "gestalt.vault.asyncio.Queue", +# side_effect=[mock_dynamic_token_queue, mock_kube_token_queue]).start() +# +# v = Vault(role=None, jwt=None) +# g = gestalt.Gestalt() +# g.add_config_file("./tests/testvault/testmount.json") +# g.configure_provider("vault", v) +# g.build_config() +# g.get_string("test_mount") +# +# mock_vault_client_read.assert_called() +# mock_dynamic_token_queue.put_nowait.assert_called() +# +# mock_vault_client_read.stop() +# mock_dynamic_token_queue.stop() +# mock_kube_token_queue.stop() +# mock_queues.stop() +# mock_vault_client_read.stop() +# +# From 355568da3da270941ef586bb6e70bd0bd61db947 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:24:43 -0400 Subject: [PATCH 08/25] tests --- gestalt/__init__.py | 5 +++-- gestalt/vault.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gestalt/__init__.py b/gestalt/__init__.py index f8edee1..a4558c4 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -545,7 +545,7 @@ def dump(self) -> Text: def _get_config_for_key(self, key: str, - key_to_search, + key_to_search: str, default: Optional[Union[str, int, float, bool,List[Any]]], object_type: Type[Union[str, int, float, bool, List[Any]]]) -> Optional[Union[str, int, float, bool, List[Any]]]: if key_to_search in self.__conf_sets: @@ -568,7 +568,7 @@ def _get_config_for_key(self, if key_to_search in self.__conf_data: val = self.__conf_data[key_to_search] for provider in self.providers.values(): - if val.startswith(provider.scheme): + if isinstance(val, str) and val.startswith(provider.scheme): regex_search = self.regex_pattern.search(val) if regex_search is not None: path = regex_search.group(2) @@ -602,3 +602,4 @@ def _get_config_for_key(self, f'Given default set key is not of type {object_type}, but of type {type(val)}' ) return val + return diff --git a/gestalt/vault.py b/gestalt/vault.py index c50c579..a634883 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -11,7 +11,7 @@ from retry import retry -def _get_nested_key(requested_data: Dict, key: str, sep: str) -> Any: +def _get_nested_key(requested_data: Dict[str, Any], key: str, sep: str) -> Any: for nested_key in key.split(sep): requested_data = requested_data[nested_key] return requested_data @@ -148,7 +148,7 @@ def _is_secret_expired(self, key: str) -> bool: print(f"TTL key {key} found expired.") return is_expired - def _set_secrets_ttl(self, requested_data: Dict, key: str) -> None: + def _set_secrets_ttl(self, requested_data: Dict[str, Any], key: str) -> None: last_vault_rotation_str = requested_data["last_vault_rotation"].split(".")[0] # to the nearest second last_vault_rotation_dt = datetime.strptime(last_vault_rotation_str, '%Y-%m-%dT%H:%M:%S') ttl = requested_data["ttl"] From 38a13eade699f6098755e44684de55686d940ef9 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:29:02 -0400 Subject: [PATCH 09/25] tests --- gestalt/__init__.py | 2 +- gestalt/provider.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gestalt/__init__.py b/gestalt/__init__.py index a4558c4..43f778a 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -602,4 +602,4 @@ def _get_config_for_key(self, f'Given default set key is not of type {object_type}, but of type {type(val)}' ) return val - return + return None diff --git a/gestalt/provider.py b/gestalt/provider.py index 45e18ff..2f1b02f 100644 --- a/gestalt/provider.py +++ b/gestalt/provider.py @@ -1,5 +1,5 @@ from abc import ABCMeta, abstractmethod -from typing import Tuple, Dict, Any, Optional +from typing import Tuple, Dict, Any, Optional, Union class Provider(metaclass=ABCMeta): @@ -15,7 +15,7 @@ def __init__(self, *args: Tuple[Any], **kwargs: Dict[Any, Any]): pass @abstractmethod - def get(self, key: str, path: str, filter: str, sep: Optional[str]) -> Any: + def get(self, key: str, path: str, filter: str, sep: Optional[str]) -> Union[str, int, float, bool]: """Abstract method to get a value from the provider """ pass From c8fa6ca62f3f4da49cf9699d90e8e406a5f679a3 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:33:13 -0400 Subject: [PATCH 10/25] tests --- gestalt/provider.py | 4 ++-- gestalt/vault.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gestalt/provider.py b/gestalt/provider.py index 2f1b02f..8b99185 100644 --- a/gestalt/provider.py +++ b/gestalt/provider.py @@ -1,5 +1,5 @@ from abc import ABCMeta, abstractmethod -from typing import Tuple, Dict, Any, Optional, Union +from typing import Tuple, Dict, Any, Optional, Union, List[Any] class Provider(metaclass=ABCMeta): @@ -15,7 +15,7 @@ def __init__(self, *args: Tuple[Any], **kwargs: Dict[Any, Any]): pass @abstractmethod - def get(self, key: str, path: str, filter: str, sep: Optional[str]) -> Union[str, int, float, bool]: + def get(self, key: str, path: str, filter: str, sep: Optional[str]) -> Union[str, int, float, bool, List[Any]]: """Abstract method to get a value from the provider """ pass diff --git a/gestalt/vault.py b/gestalt/vault.py index a634883..dd5d960 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -3,7 +3,7 @@ from gestalt.provider import Provider import requests from jsonpath_ng import parse # type: ignore -from typing import Optional, Tuple, Any, Dict +from typing import Optional, Tuple, Any, Dict, Union, List import hvac # type: ignore import asyncio import os @@ -84,7 +84,7 @@ def __init__(self, kubernetes_ttl_renew.start() @retry(RuntimeError, delay=3, tries=3) # type: ignore - def get(self, key: str, path: str, filter: str, sep: Optional[str] = ".") -> Any: + def get(self, key: str, path: str, filter: str, sep: Optional[str] = ".") -> Union[str, int, float, bool, List[Any]]: """Gets secret from vault Args: key (str): key to get secret from From aab9a48ee894cefbed9706772d1fe58d23c9b180 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:35:52 -0400 Subject: [PATCH 11/25] tests --- gestalt/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/provider.py b/gestalt/provider.py index 8b99185..465f52e 100644 --- a/gestalt/provider.py +++ b/gestalt/provider.py @@ -1,5 +1,5 @@ from abc import ABCMeta, abstractmethod -from typing import Tuple, Dict, Any, Optional, Union, List[Any] +from typing import Tuple, Dict, Any, Optional, Union, List class Provider(metaclass=ABCMeta): From 78f97570928e8c0c1639a34df61cc88914d28a88 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:38:15 -0400 Subject: [PATCH 12/25] tests --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index dd5d960..58cb0b0 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -45,7 +45,7 @@ def __init__(self, cert=cert, verify=verify) self._secret_expiry_times: Dict[str, datetime] = dict() - self._secret_values: Dict[str, Any] = dict() + self._secret_values: Dict[str, Union[str, int, float, bool, List[Any]]] = dict() try: self.vault_client.is_authenticated() From f053372f5b59a99e863249292124fa99bab1dbe1 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:48:30 -0400 Subject: [PATCH 13/25] tests --- gestalt/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 58cb0b0..8cae03f 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -130,7 +130,7 @@ def get(self, key: str, path: str, filter: str, sep: Optional[str] = ".") -> Uni match = jsonpath_expression.find(secret) if len(match) == 0: print("Path returned not matches for your secret") - returned_value_from_secret = match[0].value + returned_value_from_secret: Union[str, int, float, bool, List[Any]] = match[0].value if returned_value_from_secret == "": raise RuntimeError("Gestalt Error: Empty secret!") From 714c3a8c49532a4e8a2c14ac8cbb03146e533eb9 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:49:59 -0400 Subject: [PATCH 14/25] tests --- gestalt/vault.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 8cae03f..a5be74e 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -130,7 +130,7 @@ def get(self, key: str, path: str, filter: str, sep: Optional[str] = ".") -> Uni match = jsonpath_expression.find(secret) if len(match) == 0: print("Path returned not matches for your secret") - returned_value_from_secret: Union[str, int, float, bool, List[Any]] = match[0].value + returned_value_from_secret = match[0].value if returned_value_from_secret == "": raise RuntimeError("Gestalt Error: Empty secret!") @@ -138,7 +138,7 @@ def get(self, key: str, path: str, filter: str, sep: Optional[str] = ".") -> Uni if "ttl" in requested_data: self._set_secrets_ttl(requested_data, key) - return returned_value_from_secret + return returned_value_from_secret # type: ignore def _is_secret_expired(self, key: str) -> bool: now = datetime.now() From 69ebfb5ae4d42bad47d1556f196f1965d935ad43 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 17:52:26 -0400 Subject: [PATCH 15/25] yapf --- gestalt/__init__.py | 20 +++++++++++--------- gestalt/provider.py | 3 ++- gestalt/utils.py | 8 ++++---- gestalt/vault.py | 23 +++++++++++++++++------ tests/test_vault.py | 1 - 5 files changed, 34 insertions(+), 21 deletions(-) diff --git a/gestalt/__init__.py b/gestalt/__init__.py index 43f778a..882e79c 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -149,8 +149,7 @@ def build_config(self) -> None: f'File {f} is marked as ".yaml" but cannot be read as such: {e}' ) - self.__conf_data = flatten(self.__conf_data, - sep=self.__delim_char) + self.__conf_data = flatten(self.__conf_data, sep=self.__delim_char) self.__parse_dictionary_keys(self.__conf_data) self.__parse_dictionary_keys(self.__conf_sets) @@ -392,14 +391,17 @@ def __get( raise TypeError(f'Given key is not of string type') if default and not isinstance(default, t): raise TypeError( - f'Provided default is of incorrect type {type(default)}, it should be of type {t}' + f'Provided default is of incorrect type {type(default)}, it should be of type {t}' ) split_keys = key.split(self.__delim_char) consider_keys = list() for split_key in split_keys: consider_keys.append(split_key) joined_key = ".".join(consider_keys) - config_val = self._get_config_for_key(key=key, key_to_search=joined_key, default=default, object_type=t) + config_val = self._get_config_for_key(key=key, + key_to_search=joined_key, + default=default, + object_type=t) if config_val is not None: return config_val @@ -543,11 +545,11 @@ def dump(self) -> Text: ret.update(self.__conf_sets) return str(json.dumps(ret, indent=4)) - def _get_config_for_key(self, - key: str, - key_to_search: str, - default: Optional[Union[str, int, float, bool,List[Any]]], - object_type: Type[Union[str, int, float, bool, List[Any]]]) -> Optional[Union[str, int, float, bool, List[Any]]]: + def _get_config_for_key( + self, key: str, key_to_search: str, + default: Optional[Union[str, int, float, bool, List[Any]]], + object_type: Type[Union[str, int, float, bool, List[Any]]] + ) -> Optional[Union[str, int, float, bool, List[Any]]]: if key_to_search in self.__conf_sets: val = self.__conf_sets[key_to_search] if not isinstance(val, object_type): diff --git a/gestalt/provider.py b/gestalt/provider.py index 465f52e..78acac7 100644 --- a/gestalt/provider.py +++ b/gestalt/provider.py @@ -15,7 +15,8 @@ def __init__(self, *args: Tuple[Any], **kwargs: Dict[Any, Any]): pass @abstractmethod - def get(self, key: str, path: str, filter: str, sep: Optional[str]) -> Union[str, int, float, bool, List[Any]]: + def get(self, key: str, path: str, filter: str, + sep: Optional[str]) -> Union[str, int, float, bool, List[Any]]: """Abstract method to get a value from the provider """ pass diff --git a/gestalt/utils.py b/gestalt/utils.py index 892f082..99a5f0a 100644 --- a/gestalt/utils.py +++ b/gestalt/utils.py @@ -3,9 +3,9 @@ def flatten( - d: MutableMapping[Text, Any], - parent_key: str = '', - sep: str = '.' + d: MutableMapping[Text, Any], + parent_key: str = '', + sep: str = '.' ) -> Dict[Text, Union[List[Any], Text, int, bool, float]]: items: List[Any] = [] for k, v in d.items(): @@ -14,4 +14,4 @@ def flatten( items.extend(flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) - return dict(items) \ No newline at end of file + return dict(items) diff --git a/gestalt/vault.py b/gestalt/vault.py index a5be74e..8b07bc5 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -45,7 +45,8 @@ def __init__(self, cert=cert, verify=verify) self._secret_expiry_times: Dict[str, datetime] = dict() - self._secret_values: Dict[str, Union[str, int, float, bool, List[Any]]] = dict() + self._secret_values: Dict[str, Union[str, int, float, bool, + List[Any]]] = dict() try: self.vault_client.is_authenticated() @@ -84,7 +85,13 @@ def __init__(self, kubernetes_ttl_renew.start() @retry(RuntimeError, delay=3, tries=3) # type: ignore - def get(self, key: str, path: str, filter: str, sep: Optional[str] = ".") -> Union[str, int, float, bool, List[Any]]: + def get( + self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." + ) -> Union[str, int, float, bool, List[Any]]: """Gets secret from vault Args: key (str): key to get secret from @@ -100,7 +107,8 @@ def get(self, key: str, path: str, filter: str, sep: Optional[str] = ".") -> Uni return self._secret_values[key] # if the secret can expire but hasn't expired yet - if key in self._secret_expiry_times and not self._is_secret_expired(key): + if key in self._secret_expiry_times and not self._is_secret_expired( + key): print(f"Found unexpired TTL key {key}. Not going to Vault.") return self._secret_values[key] @@ -148,9 +156,12 @@ def _is_secret_expired(self, key: str) -> bool: print(f"TTL key {key} found expired.") return is_expired - def _set_secrets_ttl(self, requested_data: Dict[str, Any], key: str) -> None: - last_vault_rotation_str = requested_data["last_vault_rotation"].split(".")[0] # to the nearest second - last_vault_rotation_dt = datetime.strptime(last_vault_rotation_str, '%Y-%m-%dT%H:%M:%S') + def _set_secrets_ttl(self, requested_data: Dict[str, Any], + key: str) -> None: + last_vault_rotation_str = requested_data["last_vault_rotation"].split( + ".")[0] # to the nearest second + last_vault_rotation_dt = datetime.strptime(last_vault_rotation_str, + '%Y-%m-%dT%H:%M:%S') ttl = requested_data["ttl"] secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) self._secret_expiry_times[key] = secret_expires_dt diff --git a/tests/test_vault.py b/tests/test_vault.py index 157b0ef..c9546f9 100644 --- a/tests/test_vault.py +++ b/tests/test_vault.py @@ -26,4 +26,3 @@ def test_get_cache_hit(mock_db_role_request): assert result_one == expected and result_two == expected assert vault._secret_expiry_times[key] == expected_expiry_time assert vault._secret_values[key] == expected - From 8ce6debeda440d0923f8880a8909c05ad3fb2a34 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 18:04:36 -0400 Subject: [PATCH 16/25] tests --- tests/test_gestalt.py | 1088 ++++++++++++++++++++--------------------- 1 file changed, 544 insertions(+), 544 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index 376fa3b..ba33182 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -1,545 +1,545 @@ -# # type: ignore -# -# from unittest.mock import patch, Mock -# -# from gestalt.vault import Vault -# from gestalt import merge_into -# import asyncio -# import pytest -# import os -# import gestalt -# import hvac -# -# -# # Testing member function -# def test_merge_into(): -# combine1 = {} -# combine2 = {} -# combine3 = {"local": 1234, "pg": {"host": "dict1_pg", "pass": "dict1_pg"}} -# combine4 = {"local": 1234, "pg": {"host": "dict2_pg"}} -# -# merge_into(combine3, combine1) -# merge_into(combine4, combine1) -# -# merge_into(combine4, combine2) -# merge_into(combine3, combine2) -# -# assert combine1 == { -# "local": 1234, -# "pg": { -# "host": "dict2_pg", -# "pass": "dict1_pg" -# } -# } -# -# assert combine2 == { -# "local": 1234, -# "pg": { -# "host": "dict1_pg", -# "pass": "dict1_pg" -# } -# } -# -# -# def test_combine_into_empty_dict(): -# combine = {} -# merge_into({}, combine) -# assert combine == {} -# -# combine = {"local": 1234} -# merge_into({}, combine) -# assert combine == {"local": 1234} -# -# -# # Testing JSON Loading -# def test_loading_json(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# x = g.dump() -# assert len(x) -# -# -# def test_loading_json_file(): -# g = gestalt.Gestalt() -# g.add_config_file('./tests/testdata/testjson.json') -# g.build_config() -# x = g.dump() -# assert len(x) -# -# -# def test_loading_file_dir(): -# g = gestalt.Gestalt() -# with pytest.raises(ValueError) as terr: -# g.add_config_file('./tests/testdata') -# assert 'is not a file' in terr -# -# -# def test_loading_file_nonexist(): -# g = gestalt.Gestalt() -# with pytest.raises(ValueError) as terr: -# g.add_config_file('./tests/testdata/nothere.yaml') -# g.build_config() -# assert 'is not a file' in terr -# -# -# def test_loading_file_bad_yaml(): -# g = gestalt.Gestalt() -# with pytest.raises(ValueError) as terr: -# g.add_config_file('./tests/testdatabad/testyaml.yaml') -# g.build_config() -# assert terr.type is ValueError -# assert 'but cannot be read as such' in terr.value.args[0] -# -# -# def test_loading_file_bad_json(): -# g = gestalt.Gestalt() -# with pytest.raises(ValueError) as terr: -# g.add_config_file('./tests/testdatabad/testjson.json') -# g.build_config() -# assert terr.type is ValueError -# assert 'but cannot be read as such' in terr.value.args[0] -# -# -# def test_loading_dir_bad_files(): -# g = gestalt.Gestalt() -# with pytest.raises(ValueError) as terr: -# g.add_config_path('./tests/testdatabad') -# g.build_config() -# assert terr.type is ValueError -# assert 'but cannot be read as such' in terr.value.args[0] -# -# -# def test_loading_dir_bad_files_yaml_only(): -# g = gestalt.Gestalt() -# with pytest.raises(ValueError) as terr: -# g.add_config_path('./tests/testdatabadyaml') -# g.build_config() -# assert terr.type is ValueError -# assert 'but cannot be read as such' in terr.value.args[0] -# -# -# def test_loading_yaml_file(): -# g = gestalt.Gestalt() -# g.add_config_file('./tests/testdata/testyaml.yaml') -# g.build_config() -# x = g.dump() -# assert len(x) -# -# -# def test_loading_json_nonexist_dir(): -# g = gestalt.Gestalt() -# with pytest.raises(ValueError) as terr: -# g.add_config_path('./nonexistpath') -# assert 'does not exist' in terr -# -# -# def test_loading_json_file_not_dir(): -# g = gestalt.Gestalt() -# with pytest.raises(ValueError) as terr: -# g.add_config_path('./setup.py') -# assert 'is not a directory' in terr -# -# -# def test_get_wrong_type(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# with pytest.raises(TypeError) as terr: -# g.get_string('numbers') -# assert 'is not of type' in terr -# -# -# def test_get_non_exist_key(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# with pytest.raises(ValueError) as terr: -# g.get_string('non-exist') -# assert 'is not in any configuration and no default is provided' in terr -# -# -# def test_get_key_wrong_type(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# with pytest.raises(TypeError) as terr: -# g.get_string(1234) -# assert 'is not of string type' in terr -# -# -# def test_get_key_wrong_default_type(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# with pytest.raises(TypeError) as terr: -# g.get_string('nonexist', 1234) -# assert 'Provided default is of incorrect type' in terr -# -# -# def test_get_json_string(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# testval = g.get_string('yarn') -# assert testval == 'blue skies' -# -# -# def test_get_json_default_string(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# testval = g.get_string('nonexist', 'mydefval') -# assert testval == 'mydefval' -# -# -# def test_get_json_set_default_string(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# g.set_default_string('nonexisttest', 'otherdefval') -# testval = g.get_string('nonexisttest') -# assert testval == 'otherdefval' -# -# -# def test_get_json_int(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# testval = g.get_int('numbers') -# assert testval == 12345678 -# -# -# def test_get_json_float(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# testval = g.get_float('strangenumbers') -# assert testval == 123.456 -# -# -# def test_get_json_bool(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# testval = g.get_bool('truthy') -# assert testval is True -# -# -# def test_get_json_list(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# testval = g.get_list('listing') -# assert testval -# assert 'dog' in testval -# assert 'cat' in testval -# -# -# def test_get_json_nested(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# testval = g.get_string('deep.nested1') -# assert testval == 'hello' -# -# -# def test_get_yaml_nested(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# testval = g.get_string('deep_yaml.nest1.nest2.foo') -# assert testval == 'hello' -# -# -# # Test Set Overriding -# def test_set_string(): -# g = gestalt.Gestalt() -# g.set_string('mykey', 'myval') -# testval = g.get_string('mykey') -# assert testval == 'myval' -# -# -# def test_set_int(): -# g = gestalt.Gestalt() -# g.set_int('mykey', 1234) -# testval = g.get_int('mykey') -# assert testval == 1234 -# -# -# def test_set_float(): -# g = gestalt.Gestalt() -# g.set_float('mykey', 45.23) -# testval = g.get_float('mykey') -# assert testval == 45.23 -# -# -# def test_set_bool(): -# g = gestalt.Gestalt() -# g.set_bool('mykey', False) -# testval = g.get_bool('mykey') -# assert testval is False -# -# -# def test_set_list(): -# g = gestalt.Gestalt() -# g.set_list('mykey', ['hi', 'bye']) -# testval = g.get_list('mykey') -# assert testval -# assert 'hi' in testval -# assert 'bye' in testval -# -# -# def test_set_int_get_bad(): -# g = gestalt.Gestalt() -# g.set_int('mykey', 1234) -# with pytest.raises(TypeError) as terr: -# g.get_string('mykey') -# assert 'Given set key is not of type' in terr -# -# -# def test_set_bad_key_type(): -# g = gestalt.Gestalt() -# with pytest.raises(TypeError) as terr: -# g.set_string(1234, 'myval') -# assert 'is not of string type' in terr -# -# -# def test_set_bad_type(): -# g = gestalt.Gestalt() -# with pytest.raises(TypeError) as terr: -# g.set_string('mykey', 123) -# assert 'is not of type' in terr -# -# -# def test_re_set_bad_type(): -# g = gestalt.Gestalt() -# g.set_string('mykey', '123') -# with pytest.raises(TypeError) as terr: -# g.set_int('mykey', 123) -# assert 'Overriding key' in terr -# -# -# def test_set_override(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# testval = g.get_int('numbers') -# assert testval == 12345678 -# g.set_int('numbers', 6543) -# testval = g.get_int('numbers') -# assert testval == 6543 -# -# -# def test_set_bad_type_file_config(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# with pytest.raises(TypeError) as terr: -# g.set_string('numbers', 'notgood') -# assert 'File config has' in terr -# -# -# def test_set_bad_type_default_config(): -# g = gestalt.Gestalt() -# g.set_default_string('mykey', 'mystring') -# with pytest.raises(TypeError) as terr: -# g.set_int('mykey', 123) -# assert 'Default config has' in terr -# -# -# # Test Env Variables -# def test_get_env_string(): -# g = gestalt.Gestalt() -# g.auto_env() -# os.environ['MYKEY'] = 'myval' -# testval = g.get_string('mykey') -# assert testval == 'myval' -# -# -# def test_get_env_int(): -# g = gestalt.Gestalt() -# g.auto_env() -# os.environ['MYKEY'] = '999' -# testval = g.get_int('mykey') -# assert testval == 999 -# -# -# def test_get_nested_env_string(): -# g = gestalt.Gestalt() -# g.auto_env() -# os.environ['MY_KEY'] = 'myval' -# testval = g.get_string('my.key') -# assert testval == 'myval' -# -# -# def test_get_env_bad_type(): -# g = gestalt.Gestalt() -# g.auto_env() -# os.environ['MY_KEY'] = 'myval' -# with pytest.raises(TypeError) as terr: -# g.get_int('my.key') -# assert "could not be converted to type" in terr -# -# -# # Test Default Values -# def test_set_default_string(): -# g = gestalt.Gestalt() -# g.set_default_string('mykey', 'myval') -# testval = g.get_string('mykey') -# assert testval == 'myval' -# -# -# def test_set_default_int(): -# g = gestalt.Gestalt() -# g.set_default_int('mykey', 1234) -# testval = g.get_int('mykey') -# assert testval == 1234 -# -# -# def test_set_default_float(): -# g = gestalt.Gestalt() -# g.set_default_float('mykey', 1234.05) -# testval = g.get_float('mykey') -# assert testval == 1234.05 -# -# -# def test_set_default_bool(): -# g = gestalt.Gestalt() -# g.set_default_bool('mykey', False) -# testval = g.get_bool('mykey') -# assert testval is False -# -# -# def test_set_default_list(): -# g = gestalt.Gestalt() -# g.set_default_list('mykey', ['bear', 'bull']) -# testval = g.get_list('mykey') -# assert testval -# assert 'bear' in testval -# assert 'bull' in testval -# -# -# def test_set_default_int_get_bad(): -# g = gestalt.Gestalt() -# g.set_default_int('mykey', 1234) -# with pytest.raises(TypeError) as terr: -# g.get_string('mykey') -# assert 'Given default set key is not of type' in terr -# -# -# def test_set_default_string_bad_key(): -# g = gestalt.Gestalt() -# with pytest.raises(TypeError) as terr: -# g.set_default_string(1234, 'myval') -# assert 'Given key is not of string type' in terr -# -# -# def test_set_default_string_bad_val(): -# g = gestalt.Gestalt() -# with pytest.raises(TypeError) as terr: -# g.set_default_string('mykey', 123) -# assert 'Input value when setting default' in terr -# -# -# def test_set_default_string_bad_val_override(): -# g = gestalt.Gestalt() -# with pytest.raises(TypeError) as terr: -# g.set_default_string('mykey', 'myval') -# g.set_default_int('mykey', 1234) -# assert 'Overriding default key' in terr -# -# -# def test_override_nested_config(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testoverride/') -# g.build_config() -# assert g.get_int("local") == 123456 -# assert g.get_string("nested1.nested2") == "final" -# assert g.get_string("pg.host") == "dev_host" -# assert g.get_string("pg.pass") == "def_pass" -# assert g.get_string("nested1.nested3.nested4.deeplevel") == "nested5" -# -# -# def test_set_default_bad_type_file_config(): -# g = gestalt.Gestalt() -# g.add_config_path('./tests/testdata') -# g.build_config() -# with pytest.raises(TypeError) as terr: -# g.set_default_string('numbers', 'notgood') -# assert 'File config has' in terr -# -# -# def test_set_default_bad_type_set_config(): -# g = gestalt.Gestalt() -# g.set_string('mykey', 'mystring') -# with pytest.raises(TypeError) as terr: -# g.set_default_int('mykey', 123) -# assert 'Set config has' in terr -# -# -# def test_vault_setup(): -# vault = Vault(role=None, jwt=None) -# assert vault.vault_client.is_authenticated() is True -# -# -# def test_vault_interpolation(secret_setup): -# g = gestalt.Gestalt() -# g.add_config_file("./tests/testvault/testcorrect.json") -# vault = Vault(role=None, jwt=None) -# g.configure_provider("vault", vault) -# g.build_config() -# secret = g.get_string("test_secret.test_secret") -# assert secret == "test_secret_password" -# -# -# def test_vault_mount_path(mount_setup): -# g = gestalt.Gestalt() -# g.add_config_file("./tests/testvault/testmount.json") -# g.configure_provider("vault", Vault(role=None, jwt=None)) -# g.build_config() -# secret = g.get_string("test_mount.test_mount") -# assert secret == "test_mount_password" -# -# -# def test_vault_incorrect_path(mount_setup): -# g = gestalt.Gestalt() -# g.add_config_file("./tests/testvault/testincorrectmount.json") -# g.configure_provider("vault", Vault(role=None, jwt=None)) -# with pytest.raises(RuntimeError): -# g.build_config() -# g.get_string("test_mount") -# -# -# def test_nest_key_for_vault(nested_setup, secret_setup): -# g = gestalt.Gestalt() -# g.add_config_file("./tests/testvault/testnested.json") -# g.configure_provider("vault", Vault(role=None, jwt=None)) -# g.build_config() -# secret_db = g.get_string("remoteAPI.database.test_secret") -# secret_slack = g.get_string("remoteAPI.slack.token") -# assert secret_db == "test_secret_password" -# assert secret_slack == "random-token" -# -# -# def test_read_no_nest_db_role(mock_db_role_request): -# g = gestalt.Gestalt() -# g.add_config_file("./tests/testvault/testsfdynamic.json") -# g.configure_provider("vault", Vault(role=None, jwt=None)) -# g.build_config() -# secret_username = g.get_string("username") -# assert secret_username == "foo" -# -# -# def test_set_vault_key(nested_setup): -# g = gestalt.Gestalt() -# g.configure_provider("vault", Vault(role=None, jwt=None)) -# g.set_string(key="test", -# value="ref+vault://secret/data/testnested#.slack.token") -# g.build_config() -# secret = g.get_string("test") -# assert secret == "ref+vault://secret/data/testnested#.slack.token" -# +# type: ignore + +from unittest.mock import patch, Mock + +from gestalt.vault import Vault +from gestalt import merge_into +import asyncio +import pytest +import os +import gestalt +import hvac + + +# Testing member function +def test_merge_into(): + combine1 = {} + combine2 = {} + combine3 = {"local": 1234, "pg": {"host": "dict1_pg", "pass": "dict1_pg"}} + combine4 = {"local": 1234, "pg": {"host": "dict2_pg"}} + + merge_into(combine3, combine1) + merge_into(combine4, combine1) + + merge_into(combine4, combine2) + merge_into(combine3, combine2) + + assert combine1 == { + "local": 1234, + "pg": { + "host": "dict2_pg", + "pass": "dict1_pg" + } + } + + assert combine2 == { + "local": 1234, + "pg": { + "host": "dict1_pg", + "pass": "dict1_pg" + } + } + + +def test_combine_into_empty_dict(): + combine = {} + merge_into({}, combine) + assert combine == {} + + combine = {"local": 1234} + merge_into({}, combine) + assert combine == {"local": 1234} + + +# Testing JSON Loading +def test_loading_json(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + x = g.dump() + assert len(x) + + +def test_loading_json_file(): + g = gestalt.Gestalt() + g.add_config_file('./tests/testdata/testjson.json') + g.build_config() + x = g.dump() + assert len(x) + + +def test_loading_file_dir(): + g = gestalt.Gestalt() + with pytest.raises(ValueError) as terr: + g.add_config_file('./tests/testdata') + assert 'is not a file' in terr + + +def test_loading_file_nonexist(): + g = gestalt.Gestalt() + with pytest.raises(ValueError) as terr: + g.add_config_file('./tests/testdata/nothere.yaml') + g.build_config() + assert 'is not a file' in terr + + +def test_loading_file_bad_yaml(): + g = gestalt.Gestalt() + with pytest.raises(ValueError) as terr: + g.add_config_file('./tests/testdatabad/testyaml.yaml') + g.build_config() + assert terr.type is ValueError + assert 'but cannot be read as such' in terr.value.args[0] + + +def test_loading_file_bad_json(): + g = gestalt.Gestalt() + with pytest.raises(ValueError) as terr: + g.add_config_file('./tests/testdatabad/testjson.json') + g.build_config() + assert terr.type is ValueError + assert 'but cannot be read as such' in terr.value.args[0] + + +def test_loading_dir_bad_files(): + g = gestalt.Gestalt() + with pytest.raises(ValueError) as terr: + g.add_config_path('./tests/testdatabad') + g.build_config() + assert terr.type is ValueError + assert 'but cannot be read as such' in terr.value.args[0] + + +def test_loading_dir_bad_files_yaml_only(): + g = gestalt.Gestalt() + with pytest.raises(ValueError) as terr: + g.add_config_path('./tests/testdatabadyaml') + g.build_config() + assert terr.type is ValueError + assert 'but cannot be read as such' in terr.value.args[0] + + +def test_loading_yaml_file(): + g = gestalt.Gestalt() + g.add_config_file('./tests/testdata/testyaml.yaml') + g.build_config() + x = g.dump() + assert len(x) + + +def test_loading_json_nonexist_dir(): + g = gestalt.Gestalt() + with pytest.raises(ValueError) as terr: + g.add_config_path('./nonexistpath') + assert 'does not exist' in terr + + +def test_loading_json_file_not_dir(): + g = gestalt.Gestalt() + with pytest.raises(ValueError) as terr: + g.add_config_path('./setup.py') + assert 'is not a directory' in terr + + +def test_get_wrong_type(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + with pytest.raises(TypeError) as terr: + g.get_string('numbers') + assert 'is not of type' in terr + + +def test_get_non_exist_key(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + with pytest.raises(ValueError) as terr: + g.get_string('non-exist') + assert 'is not in any configuration and no default is provided' in terr + + +def test_get_key_wrong_type(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + with pytest.raises(TypeError) as terr: + g.get_string(1234) + assert 'is not of string type' in terr + + +def test_get_key_wrong_default_type(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + with pytest.raises(TypeError) as terr: + g.get_string('nonexist', 1234) + assert 'Provided default is of incorrect type' in terr + + +def test_get_json_string(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + testval = g.get_string('yarn') + assert testval == 'blue skies' + + +def test_get_json_default_string(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + testval = g.get_string('nonexist', 'mydefval') + assert testval == 'mydefval' + + +def test_get_json_set_default_string(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + g.set_default_string('nonexisttest', 'otherdefval') + testval = g.get_string('nonexisttest') + assert testval == 'otherdefval' + + +def test_get_json_int(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + testval = g.get_int('numbers') + assert testval == 12345678 + + +def test_get_json_float(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + testval = g.get_float('strangenumbers') + assert testval == 123.456 + + +def test_get_json_bool(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + testval = g.get_bool('truthy') + assert testval is True + + +def test_get_json_list(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + testval = g.get_list('listing') + assert testval + assert 'dog' in testval + assert 'cat' in testval + + +def test_get_json_nested(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + testval = g.get_string('deep.nested1') + assert testval == 'hello' + + +def test_get_yaml_nested(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + testval = g.get_string('deep_yaml.nest1.nest2.foo') + assert testval == 'hello' + + +# Test Set Overriding +def test_set_string(): + g = gestalt.Gestalt() + g.set_string('mykey', 'myval') + testval = g.get_string('mykey') + assert testval == 'myval' + + +def test_set_int(): + g = gestalt.Gestalt() + g.set_int('mykey', 1234) + testval = g.get_int('mykey') + assert testval == 1234 + + +def test_set_float(): + g = gestalt.Gestalt() + g.set_float('mykey', 45.23) + testval = g.get_float('mykey') + assert testval == 45.23 + + +def test_set_bool(): + g = gestalt.Gestalt() + g.set_bool('mykey', False) + testval = g.get_bool('mykey') + assert testval is False + + +def test_set_list(): + g = gestalt.Gestalt() + g.set_list('mykey', ['hi', 'bye']) + testval = g.get_list('mykey') + assert testval + assert 'hi' in testval + assert 'bye' in testval + + +def test_set_int_get_bad(): + g = gestalt.Gestalt() + g.set_int('mykey', 1234) + with pytest.raises(TypeError) as terr: + g.get_string('mykey') + assert 'Given set key is not of type' in terr + + +def test_set_bad_key_type(): + g = gestalt.Gestalt() + with pytest.raises(TypeError) as terr: + g.set_string(1234, 'myval') + assert 'is not of string type' in terr + + +def test_set_bad_type(): + g = gestalt.Gestalt() + with pytest.raises(TypeError) as terr: + g.set_string('mykey', 123) + assert 'is not of type' in terr + + +def test_re_set_bad_type(): + g = gestalt.Gestalt() + g.set_string('mykey', '123') + with pytest.raises(TypeError) as terr: + g.set_int('mykey', 123) + assert 'Overriding key' in terr + + +def test_set_override(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + testval = g.get_int('numbers') + assert testval == 12345678 + g.set_int('numbers', 6543) + testval = g.get_int('numbers') + assert testval == 6543 + + +def test_set_bad_type_file_config(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + with pytest.raises(TypeError) as terr: + g.set_string('numbers', 'notgood') + assert 'File config has' in terr + + +def test_set_bad_type_default_config(): + g = gestalt.Gestalt() + g.set_default_string('mykey', 'mystring') + with pytest.raises(TypeError) as terr: + g.set_int('mykey', 123) + assert 'Default config has' in terr + + +# Test Env Variables +def test_get_env_string(): + g = gestalt.Gestalt() + g.auto_env() + os.environ['MYKEY'] = 'myval' + testval = g.get_string('mykey') + assert testval == 'myval' + + +def test_get_env_int(): + g = gestalt.Gestalt() + g.auto_env() + os.environ['MYKEY'] = '999' + testval = g.get_int('mykey') + assert testval == 999 + + +def test_get_nested_env_string(): + g = gestalt.Gestalt() + g.auto_env() + os.environ['MY_KEY'] = 'myval' + testval = g.get_string('my.key') + assert testval == 'myval' + + +def test_get_env_bad_type(): + g = gestalt.Gestalt() + g.auto_env() + os.environ['MY_KEY'] = 'myval' + with pytest.raises(TypeError) as terr: + g.get_int('my.key') + assert "could not be converted to type" in terr + + +# Test Default Values +def test_set_default_string(): + g = gestalt.Gestalt() + g.set_default_string('mykey', 'myval') + testval = g.get_string('mykey') + assert testval == 'myval' + + +def test_set_default_int(): + g = gestalt.Gestalt() + g.set_default_int('mykey', 1234) + testval = g.get_int('mykey') + assert testval == 1234 + + +def test_set_default_float(): + g = gestalt.Gestalt() + g.set_default_float('mykey', 1234.05) + testval = g.get_float('mykey') + assert testval == 1234.05 + + +def test_set_default_bool(): + g = gestalt.Gestalt() + g.set_default_bool('mykey', False) + testval = g.get_bool('mykey') + assert testval is False + + +def test_set_default_list(): + g = gestalt.Gestalt() + g.set_default_list('mykey', ['bear', 'bull']) + testval = g.get_list('mykey') + assert testval + assert 'bear' in testval + assert 'bull' in testval + + +def test_set_default_int_get_bad(): + g = gestalt.Gestalt() + g.set_default_int('mykey', 1234) + with pytest.raises(TypeError) as terr: + g.get_string('mykey') + assert 'Given default set key is not of type' in terr + + +def test_set_default_string_bad_key(): + g = gestalt.Gestalt() + with pytest.raises(TypeError) as terr: + g.set_default_string(1234, 'myval') + assert 'Given key is not of string type' in terr + + +def test_set_default_string_bad_val(): + g = gestalt.Gestalt() + with pytest.raises(TypeError) as terr: + g.set_default_string('mykey', 123) + assert 'Input value when setting default' in terr + + +def test_set_default_string_bad_val_override(): + g = gestalt.Gestalt() + with pytest.raises(TypeError) as terr: + g.set_default_string('mykey', 'myval') + g.set_default_int('mykey', 1234) + assert 'Overriding default key' in terr + + +def test_override_nested_config(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testoverride/') + g.build_config() + assert g.get_int("local") == 123456 + assert g.get_string("nested1.nested2") == "final" + assert g.get_string("pg.host") == "dev_host" + assert g.get_string("pg.pass") == "def_pass" + assert g.get_string("nested1.nested3.nested4.deeplevel") == "nested5" + + +def test_set_default_bad_type_file_config(): + g = gestalt.Gestalt() + g.add_config_path('./tests/testdata') + g.build_config() + with pytest.raises(TypeError) as terr: + g.set_default_string('numbers', 'notgood') + assert 'File config has' in terr + + +def test_set_default_bad_type_set_config(): + g = gestalt.Gestalt() + g.set_string('mykey', 'mystring') + with pytest.raises(TypeError) as terr: + g.set_default_int('mykey', 123) + assert 'Set config has' in terr + + +def test_vault_setup(): + vault = Vault(role=None, jwt=None) + assert vault.vault_client.is_authenticated() is True + + +def test_vault_interpolation(secret_setup): + g = gestalt.Gestalt() + g.add_config_file("./tests/testvault/testcorrect.json") + vault = Vault(role=None, jwt=None) + g.configure_provider("vault", vault) + g.build_config() + secret = g.get_string("test_secret.test_secret") + assert secret == "test_secret_password" + + +def test_vault_mount_path(mount_setup): + g = gestalt.Gestalt() + g.add_config_file("./tests/testvault/testmount.json") + g.configure_provider("vault", Vault(role=None, jwt=None)) + g.build_config() + secret = g.get_string("test_mount.test_mount") + assert secret == "test_mount_password" + + +def test_vault_incorrect_path(mount_setup): + g = gestalt.Gestalt() + g.add_config_file("./tests/testvault/testincorrectmount.json") + g.configure_provider("vault", Vault(role=None, jwt=None)) + with pytest.raises(RuntimeError): + g.build_config() + g.get_string("test_mount") + + +def test_nest_key_for_vault(nested_setup, secret_setup): + g = gestalt.Gestalt() + g.add_config_file("./tests/testvault/testnested.json") + g.configure_provider("vault", Vault(role=None, jwt=None)) + g.build_config() + secret_db = g.get_string("remoteAPI.database.test_secret") + secret_slack = g.get_string("remoteAPI.slack.token") + assert secret_db == "test_secret_password" + assert secret_slack == "random-token" + + +def test_read_no_nest_db_role(mock_db_role_request): + g = gestalt.Gestalt() + g.add_config_file("./tests/testvault/testsfdynamic.json") + g.configure_provider("vault", Vault(role=None, jwt=None)) + g.build_config() + secret_username = g.get_string("username") + assert secret_username == "foo" + + +def test_set_vault_key(nested_setup): + g = gestalt.Gestalt() + g.configure_provider("vault", Vault(role=None, jwt=None)) + g.set_string(key="test", + value="ref+vault://secret/data/testnested#.slack.token") + g.build_config() + secret = g.get_string("test") + assert secret == "ref+vault://secret/data/testnested#.slack.token" + # # @pytest.mark.asyncio # async def test_vault_worker_dynamic(mock_vault_workers, mock_vault_k8s_auth): @@ -640,5 +640,5 @@ # mock_kube_token_queue.stop() # mock_queues.stop() # mock_vault_client_read.stop() -# -# + + From d0af7c8b8fab27bc3b049615df525983a2bd5def Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 18:07:42 -0400 Subject: [PATCH 17/25] tests --- tests/test_gestalt.py | 198 +++++++++++++++++++++--------------------- 1 file changed, 98 insertions(+), 100 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index ba33182..f4c2df2 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -540,105 +540,103 @@ def test_set_vault_key(nested_setup): secret = g.get_string("test") assert secret == "ref+vault://secret/data/testnested#.slack.token" -# -# @pytest.mark.asyncio -# async def test_vault_worker_dynamic(mock_vault_workers, mock_vault_k8s_auth): -# mock_dynamic_renew, mock_k8s_renew = mock_vault_workers -# -# mock_sleep = None -# -# def except_once(self, **kwargs): -# # side effect used to exit the worker loop after one call -# if mock_sleep.call_count == 1: -# raise hvac.exceptions.VaultError("some error") -# -# with patch("gestalt.vault.sleep", side_effect=except_once, -# autospec=True) as mock_sleep: -# -# with patch("gestalt.vault.hvac.Client") as mock_client: -# v = Vault(role="test-role", jwt="test-jwt") -# -# mock_k8s_renew.start.assert_called() -# -# test_token_queue = asyncio.Queue(maxsize=0) -# await test_token_queue.put(("dynamic", 1, 100)) -# -# with pytest.raises(RuntimeError): -# await v.worker(test_token_queue) -# -# mock_sleep.assert_called() -# mock_client().sys.renew_lease.assert_called() -# mock_k8s_renew.start.assert_called_once() -# -# mock_vault_k8s_auth.stop() -# mock_dynamic_renew.stop() -# mock_k8s_renew.stop() -# -# -# @pytest.mark.asyncio -# async def test_vault_worker_k8s(mock_vault_workers): -# mock_dynamic_renew, mock_k8s_renew = mock_vault_workers -# -# mock_sleep = None -# -# def except_once(self, **kwargs): -# # side effect used to exit the worker loop after one call -# if mock_sleep.call_count == 1: -# raise hvac.exceptions.VaultError("some error") -# -# with patch("gestalt.vault.sleep", side_effect=except_once, -# autospec=True) as mock_sleep: -# with patch("gestalt.vault.hvac.Client") as mock_client: -# v = Vault(role="test-role", jwt="test-jwt") -# -# mock_k8s_renew.start.assert_called() -# -# test_token_queue = asyncio.Queue(maxsize=0) -# await test_token_queue.put(("kubernetes", 1, 100)) -# -# with pytest.raises(RuntimeError): -# await v.worker(test_token_queue) -# -# mock_sleep.assert_called() -# mock_client().auth.token.renew.assert_called() -# mock_k8s_renew.start.assert_called_once() -# -# mock_dynamic_renew.stop() -# mock_k8s_renew.stop() -# -# -# @pytest.mark.asyncio -# async def test_vault_start_dynamic_lease(mock_vault_workers): -# mock_response = { -# "lease_id": "1", -# "lease_duration": 5, -# "data": { -# "data": "mock_data" -# } -# } -# mock_vault_client_read = patch("gestalt.vault.hvac.Client.read", -# return_value=mock_response).start() -# -# mock_dynamic_token_queue = Mock() -# mock_kube_token_queue = Mock() -# mock_queues = patch( -# "gestalt.vault.asyncio.Queue", -# side_effect=[mock_dynamic_token_queue, mock_kube_token_queue]).start() -# -# v = Vault(role=None, jwt=None) -# g = gestalt.Gestalt() -# g.add_config_file("./tests/testvault/testmount.json") -# g.configure_provider("vault", v) -# g.build_config() -# g.get_string("test_mount") -# -# mock_vault_client_read.assert_called() -# mock_dynamic_token_queue.put_nowait.assert_called() -# -# mock_vault_client_read.stop() -# mock_dynamic_token_queue.stop() -# mock_kube_token_queue.stop() -# mock_queues.stop() -# mock_vault_client_read.stop() +@pytest.mark.asyncio +async def test_vault_worker_dynamic(mock_vault_workers, mock_vault_k8s_auth): + mock_dynamic_renew, mock_k8s_renew = mock_vault_workers + mock_sleep = None + + def except_once(self, **kwargs): + # side effect used to exit the worker loop after one call + if mock_sleep.call_count == 1: + raise hvac.exceptions.VaultError("some error") + + with patch("gestalt.vault.sleep", side_effect=except_once, + autospec=True) as mock_sleep: + + with patch("gestalt.vault.hvac.Client") as mock_client: + v = Vault(role="test-role", jwt="test-jwt") + + mock_k8s_renew.start.assert_called() + + test_token_queue = asyncio.Queue(maxsize=0) + await test_token_queue.put(("dynamic", 1, 100)) + + with pytest.raises(RuntimeError): + await v.worker(test_token_queue) + + mock_sleep.assert_called() + mock_client().sys.renew_lease.assert_called() + mock_k8s_renew.start.assert_called_once() + + mock_vault_k8s_auth.stop() + mock_dynamic_renew.stop() + mock_k8s_renew.stop() + + +@pytest.mark.asyncio +async def test_vault_worker_k8s(mock_vault_workers): + mock_dynamic_renew, mock_k8s_renew = mock_vault_workers + + mock_sleep = None + + def except_once(self, **kwargs): + # side effect used to exit the worker loop after one call + if mock_sleep.call_count == 1: + raise hvac.exceptions.VaultError("some error") + + with patch("gestalt.vault.sleep", side_effect=except_once, + autospec=True) as mock_sleep: + with patch("gestalt.vault.hvac.Client") as mock_client: + v = Vault(role="test-role", jwt="test-jwt") + + mock_k8s_renew.start.assert_called() + + test_token_queue = asyncio.Queue(maxsize=0) + await test_token_queue.put(("kubernetes", 1, 100)) + + with pytest.raises(RuntimeError): + await v.worker(test_token_queue) + + mock_sleep.assert_called() + mock_client().auth.token.renew.assert_called() + mock_k8s_renew.start.assert_called_once() + + mock_dynamic_renew.stop() + mock_k8s_renew.stop() + + +@pytest.mark.asyncio +async def test_vault_start_dynamic_lease(mock_vault_workers): + mock_response = { + "lease_id": "1", + "lease_duration": 5, + "data": { + "data": "mock_data" + } + } + mock_vault_client_read = patch("gestalt.vault.hvac.Client.read", + return_value=mock_response).start() + + mock_dynamic_token_queue = Mock() + mock_kube_token_queue = Mock() + mock_queues = patch( + "gestalt.vault.asyncio.Queue", + side_effect=[mock_dynamic_token_queue, mock_kube_token_queue]).start() + + v = Vault(role=None, jwt=None) + g = gestalt.Gestalt() + g.add_config_file("./tests/testvault/testmount.json") + g.configure_provider("vault", v) + g.build_config() + g.get_string("test_mount") + + mock_vault_client_read.assert_called() + mock_dynamic_token_queue.put_nowait.assert_called() + + mock_vault_client_read.stop() + mock_dynamic_token_queue.stop() + mock_kube_token_queue.stop() + mock_queues.stop() + mock_vault_client_read.stop() From c9ec0c9672590822d3aa76a6897dcd6400a3e77d Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 18:12:39 -0400 Subject: [PATCH 18/25] tests --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 52379be..6893e69 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -78,9 +78,9 @@ def nested_setup(): def mock_vault_workers(): mock_dynamic_renew = Mock() mock_k8s_renew = Mock() - patch("gestalt.vault.Thread", - side_effect=[mock_dynamic_renew, mock_k8s_renew]).start() - return (mock_dynamic_renew, mock_k8s_renew) + with patch("gestalt.vault.Thread", + side_effect=[mock_dynamic_renew, mock_k8s_renew]): + yield (mock_dynamic_renew, mock_k8s_renew) @pytest.fixture From abe530efd5e5a01566b05126d2470a735d14b8c6 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 18:15:29 -0400 Subject: [PATCH 19/25] tests --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6893e69..6b7c7f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -84,5 +84,5 @@ def mock_vault_workers(): @pytest.fixture -def mock_vault_k8s_auth(): - return patch("gestalt.vault.hvac.api.auth_methods.Kubernetes").start() +def mock_vault_k8s_auth(mocker): + mocker.patch("gestalt.vault.hvac.api.auth_methods.Kubernetes") From 1b932150e28221c0ee50a2c077cf1bd7f739cb16 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 18:22:52 -0400 Subject: [PATCH 20/25] tests --- tests/test_gestalt.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index f4c2df2..d9e8611 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -616,27 +616,28 @@ async def test_vault_start_dynamic_lease(mock_vault_workers): "data": "mock_data" } } - mock_vault_client_read = patch("gestalt.vault.hvac.Client.read", - return_value=mock_response).start() - mock_dynamic_token_queue = Mock() - mock_kube_token_queue = Mock() - mock_queues = patch( - "gestalt.vault.asyncio.Queue", - side_effect=[mock_dynamic_token_queue, mock_kube_token_queue]).start() - - v = Vault(role=None, jwt=None) - g = gestalt.Gestalt() - g.add_config_file("./tests/testvault/testmount.json") - g.configure_provider("vault", v) - g.build_config() - g.get_string("test_mount") + mock_vault_client_patch = patch("gestalt.vault.hvac.Client.read", + return_value=mock_response) + with mock_vault_client_patch as mock_vault_client_read: + mock_dynamic_token_queue = Mock() + mock_kube_token_queue = Mock() + mock_queues = patch( + "gestalt.vault.asyncio.Queue", + side_effect=[mock_dynamic_token_queue, mock_kube_token_queue]).start() + + v = Vault(role=None, jwt=None) + g = gestalt.Gestalt() + g.add_config_file("./tests/testvault/testmount.json") + g.configure_provider("vault", v) + g.build_config() + g.get_string("test_mount") - mock_vault_client_read.assert_called() - mock_dynamic_token_queue.put_nowait.assert_called() + mock_vault_client_read.assert_called() + mock_dynamic_token_queue.put_nowait.assert_called() - mock_vault_client_read.stop() - mock_dynamic_token_queue.stop() - mock_kube_token_queue.stop() - mock_queues.stop() - mock_vault_client_read.stop() + mock_vault_client_read.stop() + mock_dynamic_token_queue.stop() + mock_kube_token_queue.stop() + mock_queues.stop() + mock_vault_client_read.stop() From 5ea06bb43940b9a48528a9bbc10c39804cc81695 Mon Sep 17 00:00:00 2001 From: adisun Date: Thu, 15 Jun 2023 18:32:30 -0400 Subject: [PATCH 21/25] tests --- tests/conftest.py | 2 +- tests/test_gestalt.py | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6b7c7f5..1a3aa94 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,7 +79,7 @@ def mock_vault_workers(): mock_dynamic_renew = Mock() mock_k8s_renew = Mock() with patch("gestalt.vault.Thread", - side_effect=[mock_dynamic_renew, mock_k8s_renew]): + side_effect=[mock_dynamic_renew, mock_k8s_renew]): yield (mock_dynamic_renew, mock_k8s_renew) diff --git a/tests/test_gestalt.py b/tests/test_gestalt.py index d9e8611..a5fc2de 100644 --- a/tests/test_gestalt.py +++ b/tests/test_gestalt.py @@ -570,7 +570,6 @@ def except_once(self, **kwargs): mock_client().sys.renew_lease.assert_called() mock_k8s_renew.start.assert_called_once() - mock_vault_k8s_auth.stop() mock_dynamic_renew.stop() mock_k8s_renew.stop() @@ -622,22 +621,23 @@ async def test_vault_start_dynamic_lease(mock_vault_workers): with mock_vault_client_patch as mock_vault_client_read: mock_dynamic_token_queue = Mock() mock_kube_token_queue = Mock() - mock_queues = patch( - "gestalt.vault.asyncio.Queue", - side_effect=[mock_dynamic_token_queue, mock_kube_token_queue]).start() - - v = Vault(role=None, jwt=None) - g = gestalt.Gestalt() - g.add_config_file("./tests/testvault/testmount.json") - g.configure_provider("vault", v) - g.build_config() - g.get_string("test_mount") - - mock_vault_client_read.assert_called() - mock_dynamic_token_queue.put_nowait.assert_called() - - mock_vault_client_read.stop() - mock_dynamic_token_queue.stop() - mock_kube_token_queue.stop() - mock_queues.stop() - mock_vault_client_read.stop() + with patch( + "gestalt.vault.asyncio.Queue", + side_effect=[mock_dynamic_token_queue, + mock_kube_token_queue]) as mock_queues: + + v = Vault(role=None, jwt=None) + g = gestalt.Gestalt() + g.add_config_file("./tests/testvault/testmount.json") + g.configure_provider("vault", v) + g.build_config() + g.get_string("test_mount") + + mock_vault_client_read.assert_called() + mock_dynamic_token_queue.put_nowait.assert_called() + + mock_vault_client_read.stop() + mock_dynamic_token_queue.stop() + mock_kube_token_queue.stop() + mock_queues.stop() + mock_vault_client_read.stop() From cd15e75e467533f809c81cf7ad69b99767c2b179 Mon Sep 17 00:00:00 2001 From: adisun Date: Fri, 16 Jun 2023 11:20:55 -0400 Subject: [PATCH 22/25] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 00bd948..46644a9 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def readme(): reqs_list = list(map(lambda x: x.rstrip(), reqs)) setup(name='gestalt-cfg', - version='3.3.0', + version='3.4.0', description='A sensible configuration library for Python', long_description=readme(), long_description_content_type="text/markdown", From 6b1f5bed788d9957a13e0550cb9691fabd9bd7bc Mon Sep 17 00:00:00 2001 From: adisun Date: Fri, 16 Jun 2023 11:21:15 -0400 Subject: [PATCH 23/25] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 46644a9..00bd948 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def readme(): reqs_list = list(map(lambda x: x.rstrip(), reqs)) setup(name='gestalt-cfg', - version='3.4.0', + version='3.3.0', description='A sensible configuration library for Python', long_description=readme(), long_description_content_type="text/markdown", From 53c41b58360746d7bc5bc4612d86c41edca5e0cd Mon Sep 17 00:00:00 2001 From: adisun Date: Tue, 20 Jun 2023 13:16:03 -0400 Subject: [PATCH 24/25] add delim char in join --- gestalt/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gestalt/__init__.py b/gestalt/__init__.py index 882e79c..2562277 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -397,7 +397,7 @@ def __get( consider_keys = list() for split_key in split_keys: consider_keys.append(split_key) - joined_key = ".".join(consider_keys) + joined_key = self.__delim_char.join(consider_keys) config_val = self._get_config_for_key(key=key, key_to_search=joined_key, default=default, From ec0aaa2a7699b7aadb523244b736f1598ae99397 Mon Sep 17 00:00:00 2001 From: adisun Date: Tue, 20 Jun 2023 16:36:49 -0400 Subject: [PATCH 25/25] address comments --- gestalt/vault.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 8b07bc5..7ab141c 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -11,12 +11,6 @@ from retry import retry -def _get_nested_key(requested_data: Dict[str, Any], key: str, sep: str) -> Any: - for nested_key in key.split(sep): - requested_data = requested_data[nested_key] - return requested_data - - class Vault(Provider): @retry(exceptions=RuntimeError, delay=2, tries=5) # type: ignore def __init__(self, @@ -103,13 +97,11 @@ def get( """ # if the key has been read before and is not a TTL secret if key in self._secret_values and key not in self._secret_expiry_times: - print(f"Found key {key} in cache with no TTL. Not going to Vault.") return self._secret_values[key] # if the secret can expire but hasn't expired yet if key in self._secret_expiry_times and not self._is_secret_expired( key): - print(f"Found unexpired TTL key {key}. Not going to Vault.") return self._secret_values[key] try: @@ -130,8 +122,6 @@ def get( except Exception as err: raise RuntimeError(f"Gestalt Error: {err}") if filter is None: - # if len(key.split(sep)) > 1: - # return _get_nested_key(requested_data, key, sep) return requested_data secret = requested_data jsonpath_expression = parse(f"${filter}") @@ -152,8 +142,6 @@ def _is_secret_expired(self, key: str) -> bool: now = datetime.now() secret_expires_dt = self._secret_expiry_times[key] is_expired = now >= secret_expires_dt - if is_expired: - print(f"TTL key {key} found expired.") return is_expired def _set_secrets_ttl(self, requested_data: Dict[str, Any],