From c59ba43115343311a7692d40fad2e447f88c738e Mon Sep 17 00:00:00 2001 From: Tanishq Dubey Date: Fri, 27 Dec 2019 14:40:07 -0500 Subject: [PATCH 1/6] prep for pypi --- .github/workflows/pythonpackage.yml | 6 ++--- __init__.py => gestalt/__init__.py | 0 gestalt.py => gestalt/gestalt.py | 0 test_gestalt.py => gestalt/test_gestalt.py | 4 ++- requirements.txt | 2 ++ setup.py | 29 ++++++++++++++++++++++ 6 files changed, 37 insertions(+), 4 deletions(-) rename __init__.py => gestalt/__init__.py (100%) rename gestalt.py => gestalt/gestalt.py (100%) rename test_gestalt.py => gestalt/test_gestalt.py (99%) create mode 100644 setup.py diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 643520d..46c53b3 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -32,12 +32,12 @@ jobs: run: | pip install pytest pip install pytest-cov - pytest -s --cov=gestalt . + pytest -s --cov=gestalt gestalt - name: Typecheck with mypy run: | pip install mypy - mypy --strict gestalt.py + mypy --strict gestalt - name: Style check with yapf run: | pip install yapf - yapf -d gestalt.py + yapf -d --recursive . diff --git a/__init__.py b/gestalt/__init__.py similarity index 100% rename from __init__.py rename to gestalt/__init__.py diff --git a/gestalt.py b/gestalt/gestalt.py similarity index 100% rename from gestalt.py rename to gestalt/gestalt.py diff --git a/test_gestalt.py b/gestalt/test_gestalt.py similarity index 99% rename from test_gestalt.py rename to gestalt/test_gestalt.py index 5b6361d..ee1a7e0 100644 --- a/test_gestalt.py +++ b/gestalt/test_gestalt.py @@ -1,3 +1,5 @@ +# type: ignore + import pytest import os from gestalt import gestalt @@ -22,7 +24,7 @@ def test_loading_json_nonexist_dir(): def test_loading_json_file_not_dir(): g = gestalt.Gestalt() with pytest.raises(ValueError) as terr: - g.add_config_path('./gestalt.py') + g.add_config_path('./setup.py') assert 'is not a directory' in terr diff --git a/requirements.txt b/requirements.txt index e69de29..b9bac65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,2 @@ +mypy==0.720 +mypy-extensions==0.4.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d852d5b --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +from setuptools import setup + + +def readme(): + with open('README.md') as f: + return f.read() + + +setup(name='gestalt-cfg', + version='1.0.0', + description='A sensible configuration library for Python', + long_description=readme(), + long_description_content_type="text/markdown", + url='https://github.com/clear-street/gestalt', + author='Clear Street', + author_email='engineering@clearstreet.io', + license='MIT', + packages=['gestalt'], + install_requires=['mypy==0.720', 'mypy-extensions==0.4.1'], + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3', + 'Topic :: Software Development :: Libraries', + 'Programming Language :: Python :: 3 :: Only', + 'Operating System :: OS Independent', + ]) From 34fe81c916c9a459bfad8684acfaa1a6115820f1 Mon Sep 17 00:00:00 2001 From: Tanishq Dubey Date: Fri, 27 Dec 2019 16:08:03 -0500 Subject: [PATCH 2/6] restructure --- LICENSE | 21 + gestalt/__init__.py | 456 +++++++++++++++++++++ gestalt/gestalt.py | 456 --------------------- setup.cfg | 6 + setup.py | 3 +- tests/__init__.py | 0 {gestalt => tests}/test_gestalt.py | 34 +- {testdata => tests/testdata}/testjson.json | 0 8 files changed, 501 insertions(+), 475 deletions(-) create mode 100644 LICENSE delete mode 100644 gestalt/gestalt.py create mode 100644 setup.cfg create mode 100644 tests/__init__.py rename {gestalt => tests}/test_gestalt.py (92%) rename {testdata => tests/testdata}/testjson.json (100%) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..15bc72f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/gestalt/__init__.py b/gestalt/__init__.py index e69de29..d6a8ad5 100644 --- a/gestalt/__init__.py +++ b/gestalt/__init__.py @@ -0,0 +1,456 @@ +import os +import glob +import json +import collections.abc as collections +from typing import Dict, List, Type, Union, Optional, MutableMapping, Text, Any + + +class Gestalt: + def __init__(self) -> None: + """ Creates the default configuration manager + + The default seetings are as follows: + - JSON filetype + - Environment variables disabled + - Configuration delimiter is '.' + - No environment variables prefix + """ + self.__conf_data: Dict[Text, Union[List[Any], Text, int, bool, + float]] = dict() + self.__conf_file_format: Text = 'json' + self.__conf_file_name: Text = '*' + self.__conf_file_paths: List[str] = [] + self.__use_env: bool = False + self.__env_prefix: Text = '' + self.__delim_char: Text = '.' + self.__conf_sets: Dict[Text, Union[List[Any], Text, int, bool, + float]] = dict() + self.__conf_defaults: Dict[Text, Union[List[Any], Text, int, bool, + float]] = dict() + + def __flatten( + self, + d: MutableMapping[Text, Any], + parent_key: str = '', + sep: str = '.' + ) -> Dict[Text, Union[List[Any], Text, int, bool, float]]: + items: List[Any] = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, collections.MutableMapping): + items.extend(self.__flatten(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + def add_config_path(self, path: str) -> None: + """Adds a path to read configs from. + + Configurations are read in the order they are added in, and overlapping keys are + overwritten in that same order, so the last config path added has the highest + priority. + + The provided path argument can contain environment variables and also be relative, + the library will expand this into an absolute path. + + Args: + path (str): Path from which to load configuration files + + Raises: + ValueError: If the `path` does not exist or is it not a directory + """ + tmp = os.path.abspath(os.path.expandvars(path)) + print(tmp) + if not os.path.lexists(tmp): + raise ValueError(f'Given path of {path} does not exist') + if not os.path.isdir(tmp): + raise ValueError(f'Given path of {path} is not a directory') + self.__conf_file_paths.append(tmp) + + def build_config(self) -> None: + """Renders all configuration paths into the internal data structure + + This does not affect if environment variables are used, it just deals + with the files that need to be loaded. + """ + for p in self.__conf_file_paths: + files = glob.glob( + p + f'/{self.__conf_file_name}.{self.__conf_file_format}') + for f in files: + with open(f) as cf: + d = json.load(cf) + self.__conf_data.update(d) + self.__conf_data = self.__flatten(self.__conf_data, + sep=self.__delim_char) + + def auto_env(self) -> None: + """Auto env provides sane defaults for using environment variables + + Specifically, auto_env will enable the use of environment variables and + will also clear the prefix for environment variables. + """ + self.__use_env = True + self.__env_prefix = '' + + def __set(self, key: str, value: Union[str, int, float, bool, List[Any]], + t: Type[Union[str, int, float, bool, List[Any]]]) -> None: + if not isinstance(key, str): + raise TypeError(f'Given key is not of string type') + if not isinstance(value, t): + raise TypeError( + f'Input value when setting {t} of type {type(value)} is not permitted' + ) + if key in self.__conf_data and not isinstance(self.__conf_data[key], + t): + raise TypeError( + f'File config has {key} with type {type(self.__conf_data[key])}. \ + Setting key with type {t} is not permitted') + if key in self.__conf_defaults and not isinstance( + self.__conf_defaults[key], t): + raise TypeError( + f'Default config has {key} with type {type(self.__conf_defaults[key])}. \ + Setting key with type {t} is not permitted') + if key in self.__conf_sets and not isinstance(self.__conf_sets[key], + t): + raise TypeError( + f'Overriding key {key} with type {type(self.__conf_sets[key])} with a {t} is not permitted' + ) + self.__conf_sets[key] = value + + def set_string(self, key: str, value: str) -> None: + """Sets the override string configuration for a given key + + Args: + key (str): The key to override + value (str): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of string type. Also + raised if the key sets value for a differing type. + """ + self.__set(key, value, str) + + def set_int(self, key: str, value: int) -> None: + """Sets the override int configuration for a given key + + Args: + key (str): The key to override + value (int): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of int type. Also + raised if the key sets value for a differing type. + """ + self.__set(key, value, int) + + def set_float(self, key: str, value: float) -> None: + """Sets the override float configuration for a given key + + Args: + key (str): The key to override + value (float): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of float type. Also + raised if the key sets value for a differing type. + """ + self.__set(key, value, float) + + def set_bool(self, key: str, value: bool) -> None: + """Sets the override boolean configuration for a given key + + Args: + key (str): The key to override + value (bool): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of bool type. Also + raised if the key sets value for a differing type. + """ + self.__set(key, value, bool) + + def set_list(self, key: str, value: List[Any]) -> None: + """Sets the override list configuration for a given key + + Args: + key (str): The key to override + value (list): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of list type. Also + raised if the key sets value for a differing type. + """ + self.__set(key, value, list) + + def __set_default( + self, key: str, value: Union[str, int, float, bool, List[Any]], + t: Type[Union[str, int, float, bool, List[Any]]]) -> None: + if not isinstance(key, str): + raise TypeError(f'Given key is not of string type') + if not isinstance(value, t): + raise TypeError( + f'Input value when setting default {t} of type {type(value)} is not permitted' + ) + if key in self.__conf_data and not isinstance(self.__conf_data[key], + t): + raise TypeError( + f'File config has {key} with type {type(self.__conf_data[key])}. \ + Setting default with type {t} is not permitted') + if key in self.__conf_sets and not isinstance(self.__conf_sets[key], + t): + raise TypeError( + f'Set config has {key} with type {type(self.__conf_sets[key])}. \ + Setting key with type {t} is not permitted') + if key in self.__conf_defaults and not isinstance( + self.__conf_defaults[key], t): + raise TypeError(f'Overriding default key {key} \ + with type {type(self.__conf_defaults[key])} with a {t} is not permitted' + ) + self.__conf_defaults[key] = value + + def set_default_string(self, key: str, value: str) -> None: + """Sets the default string configuration for a given key + + Args: + key (str): The key to override + value (str): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of string type. Also + raised if the key sets default for a differing type. + """ + self.__set_default(key, value, str) + + def set_default_int(self, key: str, value: int) -> None: + """Sets the default int configuration for a given key + + Args: + key (str): The key to override + value (int): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of int type. Also + raised if the key sets default for a differing type. + """ + self.__set_default(key, value, int) + + def set_default_float(self, key: str, value: float) -> None: + """Sets the default float configuration for a given key + + Args: + key (str): The key to override + value (float): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of float type. Also + raised if the key sets default for a differing type. + """ + self.__set_default(key, value, float) + + def set_default_bool(self, key: str, value: bool) -> None: + """Sets the default boolean configuration for a given key + + Args: + key (str): The key to override + value (bool): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of bool type. Also + raised if the key sets default for a differing type. + """ + self.__set_default(key, value, bool) + + def set_default_list(self, key: str, value: List[Any]) -> None: + """Sets the default list configuration for a given key + + Args: + key (str): The key to override + value (list): The configuration value to store + + Raises: + TypeError: If the `key` is not a string or `value` is not of list type. Also + raised if the key sets default for a differing type. + """ + self.__set_default(key, value, list) + + def __get( + self, key: str, default: Optional[Union[str, int, float, bool, + List[Any]]], + t: Type[Union[str, int, float, bool, List[Any]]] + ) -> Union[str, int, float, bool, List[Any]]: + if not isinstance(key, str): + raise TypeError(f'Given key is not of string type') + if default and not isinstance(default, t): + raise TypeError( + f'Provided default is of incorrect type {type(default)}, it should be of type {t}' + ) + if key in self.__conf_sets: + val = self.__conf_sets[key] + if not isinstance(val, t): + raise TypeError( + f'Given set key is not of type {t}, but of type {type(val)}' + ) + return val + if self.__use_env: + e_key = key.upper().replace(self.__delim_char, '_') + if e_key in os.environ: + try: + return t(os.environ[e_key]) + except ValueError as e: + raise TypeError( + f'The environment variable {e_key} could not be converted to type {t}: {e}' + ) + if key in self.__conf_data: + if not isinstance(self.__conf_data[key], t): + raise TypeError( + f'The requested key of {key} is not of type {t} (it is {type(self.__conf_data[key])})' + ) + return self.__conf_data[key] + if default: + return default + if key in self.__conf_defaults: + val = self.__conf_defaults[key] + if not isinstance(val, t): + raise TypeError( + f'Given default set key is not of type {t}, but of type {type(val)}' + ) + return val + raise ValueError( + f'Given key {key} is not in any configuration and no default is provided' + ) + + def get_string(self, key: str, default: Optional[Text] = None) -> str: + """Gets the configuration string for a given key + + Args: + key (str): The key to get + default (Optional: string): Optional default value if a configuration does not exist + + Returns: + string: The string value at the given `key` + + Raises: + TypeError: If the `key` is not a string or `value` is not of string type. Raised if the + environment variable cannot be coalesced to the needed type. + ValueError: If the 'key' is not in any configuration and no default is provided + RuntimeError: If the internal value was stored with the incorrect type. This indicates + a serious library bug + """ + val: Union[Text, int, float, bool, + List[Any]] = self.__get(key, default, str) + if not isinstance(val, str): + raise RuntimeError( + f'Gestalt error: expected to return string, but got {type(val)}' + ) + return val + + def get_int(self, key: str, default: Optional[int] = None) -> int: + """Gets the configuration int for a given key + + Args: + key (str): The key to get + default (Optional: int): Optional default value if a configuration does not exist + + Returns: + int: The int value at the given `key` + + Raises: + TypeError: If the `key` is not a string or `value` is not of int type. Raised if the + environment variable cannot be coalesced to the needed type. + ValueError: If the 'key' is not in any configuration and no default is provided + RuntimeError: If the internal value was stored with the incorrect type. This indicates + a serious library bug + """ + val: Union[Text, int, float, bool, + List[Any]] = self.__get(key, default, int) + if not isinstance(val, int): + raise RuntimeError( + f'Gestalt error: expected to return string, but got {type(val)}' + ) + return val + + def get_float(self, key: str, default: Optional[float] = None) -> float: + """Gets the configuration float for a given key + + Args: + key (str): The key to get + default (Optional: float): Optional default value if a configuration does not exist + + Returns: + float: The float value at the given `key` + + Raises: + TypeError: If the `key` is not a string or `value` is not of float type. Raised if the + environment variable cannot be coalesced to the needed type. + ValueError: If the 'key' is not in any configuration and no default is provided + RuntimeError: If the internal value was stored with the incorrect type. This indicates + a serious library bug + """ + val: Union[Text, int, float, bool, + List[Any]] = self.__get(key, default, float) + if not isinstance(val, float): + raise RuntimeError( + f'Gestalt error: expected to return float, but got {type(val)}' + ) + return val + + def get_bool(self, key: str, default: Optional[bool] = None) -> bool: + """Gets the configuration bool for a given key + + Args: + key (str): The key to get + default (Optional: bool): Optional default value if a configuration does not exist + + Returns: + bool: The bool value at the given `key` + + Raises: + TypeError: If the `key` is not a string or `value` is not of bool type. Raised if the + environment variable cannot be coalesced to the needed type. + ValueError: If the 'key' is not in any configuration and no default is provided + RuntimeError: If the internal value was stored with the incorrect type. This indicates + a serious library bug + """ + val: Union[Text, int, float, bool, + List[Any]] = self.__get(key, default, bool) + if not isinstance(val, bool): + raise RuntimeError( + f'Gestalt error: expected to return bool, but got {type(val)}') + return val + + def get_list(self, + key: str, + default: Optional[List[Any]] = None) -> List[Any]: + """Gets the configuration list for a given key + + Args: + key (str): The key to get + default (Optional: list): Optional default value if a configuration does not exist + + Returns: + list: The list value at the given `key` + + Raises: + TypeError: If the `key` is not a string or `value` is not of list type. Raised if the + environment variable cannot be coalesced to the needed type. + ValueError: If the 'key' is not in any configuration and no default is provided + RuntimeError: If the internal value was stored with the incorrect type. This indicates + a serious library bug + """ + val: Union[Text, int, float, bool, + List[Any]] = self.__get(key, default, list) + if not isinstance(val, list): + raise RuntimeError( + f'Gestalt error: expected to return list, but got {type(val)}') + return val + + def dump(self) -> Text: + """Formats the current set of configurations as a pretty printed JSON string + + Returns: + str: JSON string representation + """ + ret: Dict[str, Any] = self.__conf_defaults + ret.update(self.__conf_data) + ret.update(self.__conf_sets) + return str(json.dumps(ret, indent=4)) diff --git a/gestalt/gestalt.py b/gestalt/gestalt.py deleted file mode 100644 index d6a8ad5..0000000 --- a/gestalt/gestalt.py +++ /dev/null @@ -1,456 +0,0 @@ -import os -import glob -import json -import collections.abc as collections -from typing import Dict, List, Type, Union, Optional, MutableMapping, Text, Any - - -class Gestalt: - def __init__(self) -> None: - """ Creates the default configuration manager - - The default seetings are as follows: - - JSON filetype - - Environment variables disabled - - Configuration delimiter is '.' - - No environment variables prefix - """ - self.__conf_data: Dict[Text, Union[List[Any], Text, int, bool, - float]] = dict() - self.__conf_file_format: Text = 'json' - self.__conf_file_name: Text = '*' - self.__conf_file_paths: List[str] = [] - self.__use_env: bool = False - self.__env_prefix: Text = '' - self.__delim_char: Text = '.' - self.__conf_sets: Dict[Text, Union[List[Any], Text, int, bool, - float]] = dict() - self.__conf_defaults: Dict[Text, Union[List[Any], Text, int, bool, - float]] = dict() - - def __flatten( - self, - d: MutableMapping[Text, Any], - parent_key: str = '', - sep: str = '.' - ) -> Dict[Text, Union[List[Any], Text, int, bool, float]]: - items: List[Any] = [] - for k, v in d.items(): - new_key = parent_key + sep + k if parent_key else k - if isinstance(v, collections.MutableMapping): - items.extend(self.__flatten(v, new_key, sep=sep).items()) - else: - items.append((new_key, v)) - return dict(items) - - def add_config_path(self, path: str) -> None: - """Adds a path to read configs from. - - Configurations are read in the order they are added in, and overlapping keys are - overwritten in that same order, so the last config path added has the highest - priority. - - The provided path argument can contain environment variables and also be relative, - the library will expand this into an absolute path. - - Args: - path (str): Path from which to load configuration files - - Raises: - ValueError: If the `path` does not exist or is it not a directory - """ - tmp = os.path.abspath(os.path.expandvars(path)) - print(tmp) - if not os.path.lexists(tmp): - raise ValueError(f'Given path of {path} does not exist') - if not os.path.isdir(tmp): - raise ValueError(f'Given path of {path} is not a directory') - self.__conf_file_paths.append(tmp) - - def build_config(self) -> None: - """Renders all configuration paths into the internal data structure - - This does not affect if environment variables are used, it just deals - with the files that need to be loaded. - """ - for p in self.__conf_file_paths: - files = glob.glob( - p + f'/{self.__conf_file_name}.{self.__conf_file_format}') - for f in files: - with open(f) as cf: - d = json.load(cf) - self.__conf_data.update(d) - self.__conf_data = self.__flatten(self.__conf_data, - sep=self.__delim_char) - - def auto_env(self) -> None: - """Auto env provides sane defaults for using environment variables - - Specifically, auto_env will enable the use of environment variables and - will also clear the prefix for environment variables. - """ - self.__use_env = True - self.__env_prefix = '' - - def __set(self, key: str, value: Union[str, int, float, bool, List[Any]], - t: Type[Union[str, int, float, bool, List[Any]]]) -> None: - if not isinstance(key, str): - raise TypeError(f'Given key is not of string type') - if not isinstance(value, t): - raise TypeError( - f'Input value when setting {t} of type {type(value)} is not permitted' - ) - if key in self.__conf_data and not isinstance(self.__conf_data[key], - t): - raise TypeError( - f'File config has {key} with type {type(self.__conf_data[key])}. \ - Setting key with type {t} is not permitted') - if key in self.__conf_defaults and not isinstance( - self.__conf_defaults[key], t): - raise TypeError( - f'Default config has {key} with type {type(self.__conf_defaults[key])}. \ - Setting key with type {t} is not permitted') - if key in self.__conf_sets and not isinstance(self.__conf_sets[key], - t): - raise TypeError( - f'Overriding key {key} with type {type(self.__conf_sets[key])} with a {t} is not permitted' - ) - self.__conf_sets[key] = value - - def set_string(self, key: str, value: str) -> None: - """Sets the override string configuration for a given key - - Args: - key (str): The key to override - value (str): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of string type. Also - raised if the key sets value for a differing type. - """ - self.__set(key, value, str) - - def set_int(self, key: str, value: int) -> None: - """Sets the override int configuration for a given key - - Args: - key (str): The key to override - value (int): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of int type. Also - raised if the key sets value for a differing type. - """ - self.__set(key, value, int) - - def set_float(self, key: str, value: float) -> None: - """Sets the override float configuration for a given key - - Args: - key (str): The key to override - value (float): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of float type. Also - raised if the key sets value for a differing type. - """ - self.__set(key, value, float) - - def set_bool(self, key: str, value: bool) -> None: - """Sets the override boolean configuration for a given key - - Args: - key (str): The key to override - value (bool): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of bool type. Also - raised if the key sets value for a differing type. - """ - self.__set(key, value, bool) - - def set_list(self, key: str, value: List[Any]) -> None: - """Sets the override list configuration for a given key - - Args: - key (str): The key to override - value (list): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of list type. Also - raised if the key sets value for a differing type. - """ - self.__set(key, value, list) - - def __set_default( - self, key: str, value: Union[str, int, float, bool, List[Any]], - t: Type[Union[str, int, float, bool, List[Any]]]) -> None: - if not isinstance(key, str): - raise TypeError(f'Given key is not of string type') - if not isinstance(value, t): - raise TypeError( - f'Input value when setting default {t} of type {type(value)} is not permitted' - ) - if key in self.__conf_data and not isinstance(self.__conf_data[key], - t): - raise TypeError( - f'File config has {key} with type {type(self.__conf_data[key])}. \ - Setting default with type {t} is not permitted') - if key in self.__conf_sets and not isinstance(self.__conf_sets[key], - t): - raise TypeError( - f'Set config has {key} with type {type(self.__conf_sets[key])}. \ - Setting key with type {t} is not permitted') - if key in self.__conf_defaults and not isinstance( - self.__conf_defaults[key], t): - raise TypeError(f'Overriding default key {key} \ - with type {type(self.__conf_defaults[key])} with a {t} is not permitted' - ) - self.__conf_defaults[key] = value - - def set_default_string(self, key: str, value: str) -> None: - """Sets the default string configuration for a given key - - Args: - key (str): The key to override - value (str): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of string type. Also - raised if the key sets default for a differing type. - """ - self.__set_default(key, value, str) - - def set_default_int(self, key: str, value: int) -> None: - """Sets the default int configuration for a given key - - Args: - key (str): The key to override - value (int): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of int type. Also - raised if the key sets default for a differing type. - """ - self.__set_default(key, value, int) - - def set_default_float(self, key: str, value: float) -> None: - """Sets the default float configuration for a given key - - Args: - key (str): The key to override - value (float): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of float type. Also - raised if the key sets default for a differing type. - """ - self.__set_default(key, value, float) - - def set_default_bool(self, key: str, value: bool) -> None: - """Sets the default boolean configuration for a given key - - Args: - key (str): The key to override - value (bool): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of bool type. Also - raised if the key sets default for a differing type. - """ - self.__set_default(key, value, bool) - - def set_default_list(self, key: str, value: List[Any]) -> None: - """Sets the default list configuration for a given key - - Args: - key (str): The key to override - value (list): The configuration value to store - - Raises: - TypeError: If the `key` is not a string or `value` is not of list type. Also - raised if the key sets default for a differing type. - """ - self.__set_default(key, value, list) - - def __get( - self, key: str, default: Optional[Union[str, int, float, bool, - List[Any]]], - t: Type[Union[str, int, float, bool, List[Any]]] - ) -> Union[str, int, float, bool, List[Any]]: - if not isinstance(key, str): - raise TypeError(f'Given key is not of string type') - if default and not isinstance(default, t): - raise TypeError( - f'Provided default is of incorrect type {type(default)}, it should be of type {t}' - ) - if key in self.__conf_sets: - val = self.__conf_sets[key] - if not isinstance(val, t): - raise TypeError( - f'Given set key is not of type {t}, but of type {type(val)}' - ) - return val - if self.__use_env: - e_key = key.upper().replace(self.__delim_char, '_') - if e_key in os.environ: - try: - return t(os.environ[e_key]) - except ValueError as e: - raise TypeError( - f'The environment variable {e_key} could not be converted to type {t}: {e}' - ) - if key in self.__conf_data: - if not isinstance(self.__conf_data[key], t): - raise TypeError( - f'The requested key of {key} is not of type {t} (it is {type(self.__conf_data[key])})' - ) - return self.__conf_data[key] - if default: - return default - if key in self.__conf_defaults: - val = self.__conf_defaults[key] - if not isinstance(val, t): - raise TypeError( - f'Given default set key is not of type {t}, but of type {type(val)}' - ) - return val - raise ValueError( - f'Given key {key} is not in any configuration and no default is provided' - ) - - def get_string(self, key: str, default: Optional[Text] = None) -> str: - """Gets the configuration string for a given key - - Args: - key (str): The key to get - default (Optional: string): Optional default value if a configuration does not exist - - Returns: - string: The string value at the given `key` - - Raises: - TypeError: If the `key` is not a string or `value` is not of string type. Raised if the - environment variable cannot be coalesced to the needed type. - ValueError: If the 'key' is not in any configuration and no default is provided - RuntimeError: If the internal value was stored with the incorrect type. This indicates - a serious library bug - """ - val: Union[Text, int, float, bool, - List[Any]] = self.__get(key, default, str) - if not isinstance(val, str): - raise RuntimeError( - f'Gestalt error: expected to return string, but got {type(val)}' - ) - return val - - def get_int(self, key: str, default: Optional[int] = None) -> int: - """Gets the configuration int for a given key - - Args: - key (str): The key to get - default (Optional: int): Optional default value if a configuration does not exist - - Returns: - int: The int value at the given `key` - - Raises: - TypeError: If the `key` is not a string or `value` is not of int type. Raised if the - environment variable cannot be coalesced to the needed type. - ValueError: If the 'key' is not in any configuration and no default is provided - RuntimeError: If the internal value was stored with the incorrect type. This indicates - a serious library bug - """ - val: Union[Text, int, float, bool, - List[Any]] = self.__get(key, default, int) - if not isinstance(val, int): - raise RuntimeError( - f'Gestalt error: expected to return string, but got {type(val)}' - ) - return val - - def get_float(self, key: str, default: Optional[float] = None) -> float: - """Gets the configuration float for a given key - - Args: - key (str): The key to get - default (Optional: float): Optional default value if a configuration does not exist - - Returns: - float: The float value at the given `key` - - Raises: - TypeError: If the `key` is not a string or `value` is not of float type. Raised if the - environment variable cannot be coalesced to the needed type. - ValueError: If the 'key' is not in any configuration and no default is provided - RuntimeError: If the internal value was stored with the incorrect type. This indicates - a serious library bug - """ - val: Union[Text, int, float, bool, - List[Any]] = self.__get(key, default, float) - if not isinstance(val, float): - raise RuntimeError( - f'Gestalt error: expected to return float, but got {type(val)}' - ) - return val - - def get_bool(self, key: str, default: Optional[bool] = None) -> bool: - """Gets the configuration bool for a given key - - Args: - key (str): The key to get - default (Optional: bool): Optional default value if a configuration does not exist - - Returns: - bool: The bool value at the given `key` - - Raises: - TypeError: If the `key` is not a string or `value` is not of bool type. Raised if the - environment variable cannot be coalesced to the needed type. - ValueError: If the 'key' is not in any configuration and no default is provided - RuntimeError: If the internal value was stored with the incorrect type. This indicates - a serious library bug - """ - val: Union[Text, int, float, bool, - List[Any]] = self.__get(key, default, bool) - if not isinstance(val, bool): - raise RuntimeError( - f'Gestalt error: expected to return bool, but got {type(val)}') - return val - - def get_list(self, - key: str, - default: Optional[List[Any]] = None) -> List[Any]: - """Gets the configuration list for a given key - - Args: - key (str): The key to get - default (Optional: list): Optional default value if a configuration does not exist - - Returns: - list: The list value at the given `key` - - Raises: - TypeError: If the `key` is not a string or `value` is not of list type. Raised if the - environment variable cannot be coalesced to the needed type. - ValueError: If the 'key' is not in any configuration and no default is provided - RuntimeError: If the internal value was stored with the incorrect type. This indicates - a serious library bug - """ - val: Union[Text, int, float, bool, - List[Any]] = self.__get(key, default, list) - if not isinstance(val, list): - raise RuntimeError( - f'Gestalt error: expected to return list, but got {type(val)}') - return val - - def dump(self) -> Text: - """Formats the current set of configurations as a pretty printed JSON string - - Returns: - str: JSON string representation - """ - ret: Dict[str, Any] = self.__conf_defaults - ret.update(self.__conf_data) - ret.update(self.__conf_sets) - return str(json.dumps(ret, indent=4)) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..f5e8c37 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[aliases] +test = pytest + +[tool:pytest] +addopts = --verbose +python_files = tests/*.py diff --git a/setup.py b/setup.py index d852d5b..dcc4ac9 100644 --- a/setup.py +++ b/setup.py @@ -16,14 +16,13 @@ def readme(): author_email='engineering@clearstreet.io', license='MIT', packages=['gestalt'], - install_requires=['mypy==0.720', 'mypy-extensions==0.4.1'], python_requires='>=3.6', classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3', 'Topic :: Software Development :: Libraries', + 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3 :: Only', 'Operating System :: OS Independent', ]) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gestalt/test_gestalt.py b/tests/test_gestalt.py similarity index 92% rename from gestalt/test_gestalt.py rename to tests/test_gestalt.py index ee1a7e0..e05f10f 100644 --- a/gestalt/test_gestalt.py +++ b/tests/test_gestalt.py @@ -2,13 +2,13 @@ import pytest import os -from gestalt import gestalt +import gestalt # Testing JSON Loading def test_loading_json(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() x = g.dump() assert len(x) @@ -30,7 +30,7 @@ def test_loading_json_file_not_dir(): def test_get_wrong_type(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() with pytest.raises(TypeError) as terr: g.get_string('numbers') @@ -39,7 +39,7 @@ def test_get_wrong_type(): def test_get_non_exist_key(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() with pytest.raises(ValueError) as terr: g.get_string('non-exist') @@ -48,7 +48,7 @@ def test_get_non_exist_key(): def test_get_key_wrong_type(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() with pytest.raises(TypeError) as terr: g.get_string(1234) @@ -57,7 +57,7 @@ def test_get_key_wrong_type(): def test_get_key_wrong_default_type(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() with pytest.raises(TypeError) as terr: g.get_string('nonexist', 1234) @@ -66,7 +66,7 @@ def test_get_key_wrong_default_type(): def test_get_json_string(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() testval = g.get_string('yarn') assert testval == 'blue skies' @@ -74,7 +74,7 @@ def test_get_json_string(): def test_get_json_default_string(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() testval = g.get_string('nonexist', 'mydefval') assert testval == 'mydefval' @@ -82,7 +82,7 @@ def test_get_json_default_string(): def test_get_json_set_default_string(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() g.set_default_string('nonexisttest', 'otherdefval') testval = g.get_string('nonexisttest') @@ -91,7 +91,7 @@ def test_get_json_set_default_string(): def test_get_json_int(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() testval = g.get_int('numbers') assert testval == 12345678 @@ -99,7 +99,7 @@ def test_get_json_int(): def test_get_json_float(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() testval = g.get_float('strangenumbers') assert testval == 123.456 @@ -107,7 +107,7 @@ def test_get_json_float(): def test_get_json_bool(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() testval = g.get_bool('truthy') assert testval is True @@ -115,7 +115,7 @@ def test_get_json_bool(): def test_get_json_list(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() testval = g.get_list('listing') assert testval @@ -125,7 +125,7 @@ def test_get_json_list(): def test_get_json_nested(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() testval = g.get_string('deep.nested1') assert testval == 'hello' @@ -201,7 +201,7 @@ def test_re_set_bad_type(): def test_set_override(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() testval = g.get_int('numbers') assert testval == 12345678 @@ -212,7 +212,7 @@ def test_set_override(): def test_set_bad_type_file_config(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() with pytest.raises(TypeError) as terr: g.set_string('numbers', 'notgood') @@ -331,7 +331,7 @@ def test_set_default_string_bad_val_override(): def test_set_default_bad_type_file_config(): g = gestalt.Gestalt() - g.add_config_path('./testdata') + g.add_config_path('./tests/testdata') g.build_config() with pytest.raises(TypeError) as terr: g.set_default_string('numbers', 'notgood') diff --git a/testdata/testjson.json b/tests/testdata/testjson.json similarity index 100% rename from testdata/testjson.json rename to tests/testdata/testjson.json From 7af6169ca636f44b9ca0c3ba2e1b7c505a732f3d Mon Sep 17 00:00:00 2001 From: Tanishq Dubey Date: Fri, 27 Dec 2019 16:09:15 -0500 Subject: [PATCH 3/6] fix cicd --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 46c53b3..f916aa8 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -32,7 +32,7 @@ jobs: run: | pip install pytest pip install pytest-cov - pytest -s --cov=gestalt gestalt + pytest -s --cov=gestalt tests/*.py - name: Typecheck with mypy run: | pip install mypy From b3129df2d294d67b0311968be546365353226b08 Mon Sep 17 00:00:00 2001 From: Tanishq Dubey Date: Fri, 27 Dec 2019 16:12:33 -0500 Subject: [PATCH 4/6] add test requirements --- .github/workflows/pythonpackage.yml | 6 +----- requirements.test.txt | 6 ++++++ 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 requirements.test.txt diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index f916aa8..5c2903a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -21,23 +21,19 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r requirements.test.txt - name: Lint with flake8 run: | - pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=15 --max-line-length=127 --statistics - name: Test with pytest run: | - pip install pytest - pip install pytest-cov pytest -s --cov=gestalt tests/*.py - name: Typecheck with mypy run: | - pip install mypy mypy --strict gestalt - name: Style check with yapf run: | - pip install yapf yapf -d --recursive . diff --git a/requirements.test.txt b/requirements.test.txt new file mode 100644 index 0000000..d079dd1 --- /dev/null +++ b/requirements.test.txt @@ -0,0 +1,6 @@ +yapf==0.29.0 +flake8==3.7.9 +mypy==0.720 +mypy-extensions==0.4.1 +pytest==4.3.1 +pytest-cov==2.8.1 From 5b46ca3c2b2f69857ca9330a86b3e5ee91b0cde0 Mon Sep 17 00:00:00 2001 From: Tanishq Dubey Date: Fri, 27 Dec 2019 16:22:57 -0500 Subject: [PATCH 5/6] minor --- requirements.txt | 2 -- setup.py | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b9bac65..e69de29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +0,0 @@ -mypy==0.720 -mypy-extensions==0.4.1 diff --git a/setup.py b/setup.py index dcc4ac9..40c4904 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,9 @@ def readme(): 'License :: OSI Approved :: MIT License', 'Topic :: Software Development :: Libraries', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3 :: Only', 'Operating System :: OS Independent', ]) From b922226f1571a3307cb2d04ba96b690a6a789d3f Mon Sep 17 00:00:00 2001 From: Tanishq Dubey Date: Mon, 6 Jan 2020 17:04:35 -0500 Subject: [PATCH 6/6] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 15bc72f..42b6e3f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 +Copyright (c) 2019 Clear Street Technologies Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal