diff --git a/.coveragerc b/.coveragerc index d859a006..c91eece7 100644 --- a/.coveragerc +++ b/.coveragerc @@ -9,6 +9,9 @@ exclude_lines = # Have to re-enable the standard pragma pragma: no cover + # Conditional code which is dependent on the OS, or `os.name` + if name == 'nt': + # Don't complain if tests don't hit defensive assertion code: raise AssertionError raise NotImplementedError diff --git a/.env b/.env new file mode 100644 index 00000000..37e195a2 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +# These values are used in unit tests (tests/unit/test_env_wizard.py) +MY_STR=42 +my_time=15:20 +MyDate=2022-01-21 diff --git a/.gitignore b/.gitignore index c783fc2d..dd5e54b7 100644 --- a/.gitignore +++ b/.gitignore @@ -88,7 +88,7 @@ target/ Pipfile.lock # Environments -.env +.env/ .venv env/ venv/ diff --git a/README.rst b/README.rst index c3a005be..4de0e0fa 100644 --- a/README.rst +++ b/README.rst @@ -101,6 +101,8 @@ Here are the key features that ``dataclass-wizard`` offers: - *Flexible (de)serialization*: Marshal dataclasses to/from JSON, TOML, YAML, or ``dict``. - *Field properties*: Use properties with default values in dataclass instances. - *JSON to Dataclass generation*: Auto-generate a dataclass schema from a JSON file or string. +- *Environment support*: easily load ``dotenv`` files and environment variables + as strongly-typed class fields. Wizard Mixins @@ -109,6 +111,7 @@ Wizard Mixins In addition to ``JSONWizard``, these handy Mixin_ classes simplify your workflow: * `JSONPyWizard`_ — A ``JSONWizard`` helper to skip *camelCase* and keep keys as-is. +* `EnvWizard`_ -- Enables loading of Environment variables and ``.env`` files into strongly-typed class schemas. * `JSONListWizard`_ -- Extends ``JSONWizard`` to return `Container`_ -- instead of *list* -- objects where possible. * `JSONFileWizard`_ -- Makes it easier to convert dataclass instances from/to JSON files on a local drive. * `TOMLWizard`_ -- Provides support to convert dataclass instances to/from TOML. @@ -1242,6 +1245,7 @@ This package was created with Cookiecutter_ and the `rnag/cookiecutter-pypackage .. _`Contributing`: https://dataclass-wizard.readthedocs.io/en/latest/contributing.html .. _`open an issue`: https://github.com/rnag/dataclass-wizard/issues .. _`JSONPyWizard`: https://dataclass-wizard.readthedocs.io/en/latest/common_use_cases/wizard_mixins.html#jsonpywizard +.. _`EnvWizard`: https://dataclass-wizard.readthedocs.io/en/latest/common_use_cases/wizard_mixins.html#envwizard .. _`JSONListWizard`: https://dataclass-wizard.readthedocs.io/en/latest/common_use_cases/wizard_mixins.html#jsonlistwizard .. _`JSONFileWizard`: https://dataclass-wizard.readthedocs.io/en/latest/common_use_cases/wizard_mixins.html#jsonfilewizard .. _`TOMLWizard`: https://dataclass-wizard.readthedocs.io/en/latest/common_use_cases/wizard_mixins.html#tomlwizard diff --git a/dataclass_wizard/__init__.py b/dataclass_wizard/__init__.py index f972c5de..0485501e 100644 --- a/dataclass_wizard/__init__.py +++ b/dataclass_wizard/__init__.py @@ -77,6 +77,7 @@ 'DumpMixin', 'property_wizard', # Wizard Mixins + 'EnvWizard', 'JSONListWizard', 'JSONFileWizard', 'TOMLWizard', @@ -123,6 +124,7 @@ Pattern, DatePattern, TimePattern, DateTimePattern, CatchAll, SkipIf, SkipIfNone, EQ, NE, LT, LE, GT, GE, IS, IS_NOT, IS_TRUTHY, IS_FALSY) +from .environ.wizard import EnvWizard from .property_wizard import property_wizard from .serial_json import JSONWizard, JSONPyWizard, JSONSerializable from .wizard_mixins import JSONListWizard, JSONFileWizard, TOMLWizard, YAMLWizard diff --git a/dataclass_wizard/abstractions.py b/dataclass_wizard/abstractions.py index e30e21a8..c1a9ac1e 100644 --- a/dataclass_wizard/abstractions.py +++ b/dataclass_wizard/abstractions.py @@ -2,10 +2,12 @@ Contains implementations for Abstract Base Classes """ import json + from abc import ABC, abstractmethod -from dataclasses import dataclass, InitVar +from dataclasses import dataclass, InitVar, Field from typing import Type, TypeVar, Dict, Generic +from .bases import META from .models import Extras from .type_def import T, TT @@ -13,7 +15,33 @@ # Create a generic variable that can be 'AbstractJSONWizard', or any subclass. W = TypeVar('W', bound='AbstractJSONWizard') -FieldToParser = Dict[str, 'AbstractParser'] + +class AbstractEnvWizard(ABC): + """ + Abstract class that defines the methods a sub-class must implement at a + minimum to be considered a "true" Environment Wizard. + """ + __slots__ = () + + # Extends the `__annotations__` attribute to return only the fields + # (variables) of the `EnvWizard` subclass. + # + # .. NOTE:: + # This excludes fields marked as ``ClassVar``, or ones which are + # not type-annotated. + __fields__: dict[str, Field] + + def dict(self): + + return {f: getattr(self, f) for f in self.__class__.__fields__} + + @abstractmethod + def to_dict(self): + ... + + @abstractmethod + def to_json(self, indent=None): + ... class AbstractJSONWizard(ABC): @@ -204,6 +232,17 @@ def load_to_date(o, base_type): def load_to_timedelta(o, base_type): ... + # @staticmethod + # @abstractmethod + # def load_func_for_dataclass( + # cls: Type[T], + # config: Optional[META], + # ) -> Callable[[JSONObject], T]: + # """ + # Generate and return the load function for a (nested) dataclass of + # type `cls`. + # """ + @classmethod @abstractmethod def get_parser_for_annotation(cls, ann_type, diff --git a/dataclass_wizard/abstractions.pyi b/dataclass_wizard/abstractions.pyi index 97e06c3d..4850ae89 100644 --- a/dataclass_wizard/abstractions.pyi +++ b/dataclass_wizard/abstractions.pyi @@ -3,7 +3,7 @@ Contains implementations for Abstract Base Classes """ import json from abc import ABC, abstractmethod -from dataclasses import dataclass, InitVar +from dataclasses import dataclass, InitVar, Field from datetime import datetime, time, date, timedelta from decimal import Decimal from typing import ( @@ -18,10 +18,53 @@ from .type_def import ( ) +# Create a generic variable that can be 'AbstractEnvWizard', or any subclass. +E = TypeVar('E', bound='AbstractEnvWizard') + # Create a generic variable that can be 'AbstractJSONWizard', or any subclass. W = TypeVar('W', bound='AbstractJSONWizard') -FieldToParser = dict[str, 'AbstractParser'] +FieldToParser = dict[str, AbstractParser] + + +class AbstractEnvWizard(ABC): + """ + Abstract class that defines the methods a sub-class must implement at a + minimum to be considered a "true" Environment Wizard. + """ + __slots__ = () + + # Extends the `__annotations__` attribute to return only the fields + # (variables) of the `EnvWizard` subclass. + # + # .. NOTE:: + # This excludes fields marked as ``ClassVar``, or ones which are + # not type-annotated. + __fields__: dict[str, Field] + + def dict(self: E) -> JSONObject: + """ + Same as ``__dict__``, but only returns values for fields defined + on the `EnvWizard` instance. See :attr:`__fields__` for more info. + + .. NOTE:: + The values in the returned dictionary object are not needed to be + JSON serializable. Use :meth:`to_dict` if this is required. + """ + + @abstractmethod + def to_dict(self: E) -> JSONObject: + """ + Converts an instance of a `EnvWizard` subclass to a Python dictionary + object that is JSON serializable. + """ + + @abstractmethod + def to_json(self: E, indent=None) -> AnyStr: + """ + Converts an instance of a `EnvWizard` subclass to a JSON `string` + representation. + """ class AbstractJSONWizard(ABC): @@ -127,7 +170,6 @@ class AbstractParser(ABC, Generic[T, TT]): type. Checks against the exact type instead of `isinstance` so we can handle special cases like `bool`, which is a subclass of `int`. """ - return type(item) is self.base_type @abstractmethod def __call__(self, o: Any) -> TT: diff --git a/dataclass_wizard/bases.py b/dataclass_wizard/bases.py index 5607c8ce..96f83aa3 100644 --- a/dataclass_wizard/bases.py +++ b/dataclass_wizard/bases.py @@ -3,9 +3,9 @@ from .constants import TAG from .decorators import cached_class_property -from .enums import DateTimeTo, LetterCase from .models import Condition -from .type_def import FrozenKeys +from .enums import DateTimeTo, LetterCase, LetterCasePriority +from .type_def import FrozenKeys, EnvFileType # Create a generic variable that can be 'AbstractMeta', or any subclass. @@ -249,6 +249,100 @@ def bind_to(cls, dataclass: Type, create=True, is_default=True): """ +class AbstractEnvMeta: + """ + Base class definition for the `EnvWizard.Meta` inner class. + """ + __slots__ = () + + # A list of class attributes that are exclusive to the Meta config. + # When merging two Meta configs for a class, these are the only + # attributes which will *not* be merged. + __special_attrs__ = frozenset({ + 'debug_enabled', + 'env_var_to_field', + }) + + # Class attribute which enables us to detect a `EnvWizard.Meta` subclass. + __is_inner_meta__ = False + + # True to enable Debug mode for additional (more verbose) log output. + # + # For example, a message is logged with the environment variable that is + # mapped to each attribute. + # + # This also results in more helpful messages during error handling, which + # can be useful when debugging the cause when values are an invalid type + # (i.e. they don't match the annotation for the field) when unmarshalling + # a environ variable values to attributes in an EnvWizard subclass. + # + # Note there is a minor performance impact when DEBUG mode is enabled. + debug_enabled: ClassVar[bool] = False + + # `True` to load environment variables from an `.env` file, or a + # list/tuple of dotenv files. + # + # This can also be set to a path to a custom dotenv file, for example: + # `path/to/.env.prod` + # + # Simply passing in a filename such as `.env.prod` will search the current + # directory, as well as any parent folders (working backwards to the root + # directory), until it locates the given file. + # + # If multiple files are passed in, later files in the list/tuple will take + # priority over earlier files. + # + # For example, in below the '.env.last' file takes priority over '.env': + # env_file = '.env', '.env.last' + env_file: ClassVar[EnvFileType] = None + + # A customized mapping of field in the `EnvWizard` subclass to its + # corresponding environment variable to search for. + # + # Note: this is in addition to the implicit field transformations, like + # "myStr" -> "my_str" + field_to_env_var: ClassVar[Dict[str, str]] = None + + # The letter casing priority to use when looking up Env Var Names. + # + # The default is `SCREAMING_SNAKE_CASE`. + key_lookup_with_load: ClassVar[Union[LetterCasePriority, str]] = LetterCasePriority.SCREAMING_SNAKE + + # How `EnvWizard` fields (variables) should be transformed to JSON keys. + # + # The default is 'snake_case'. + key_transform_with_dump: ClassVar[Union[LetterCase, str]] = LetterCase.SNAKE + + # Determines whether we should we skip / omit fields with default values + # in the serialization process. + skip_defaults: ClassVar[bool] = False + + @cached_class_property + def all_fields(cls) -> FrozenKeys: + """Return a list of all class attributes""" + return frozenset(AbstractEnvMeta.__annotations__) + + @cached_class_property + def fields_to_merge(cls) -> FrozenKeys: + """Return a list of class attributes, minus `__special_attrs__`""" + return cls.all_fields - cls.__special_attrs__ + + @classmethod + @abstractmethod + def bind_to(cls, env_class: Type, create=True): + """ + Initialize hook which applies the Meta config to `env_class`, which is + typically a subclass of :class:`EnvWizard`. + + :param env_class: A sub-class of :class:`EnvWizard`. + :param create: When true, a separate loader/dumper will be created + for the class. If disabled, this will access the root loader/dumper, + so modifying this should affect global settings across all + dataclasses that use the JSON load/dump process. + + """ + + class BaseLoadHook: """ Container class for type hooks. diff --git a/dataclass_wizard/bases_meta.py b/dataclass_wizard/bases_meta.py index d1a1fe7c..01264ba8 100644 --- a/dataclass_wizard/bases_meta.py +++ b/dataclass_wizard/bases_meta.py @@ -9,21 +9,23 @@ from typing import Type, Optional, Dict, Union from .abstractions import AbstractJSONWizard -from .bases import AbstractMeta, META +from .bases import AbstractMeta, META, AbstractEnvMeta from .class_helper import ( META_INITIALIZER, _META, get_outer_class_name, get_class_name, create_new_class, - json_field_to_dataclass_field, dataclass_field_to_json_field + json_field_to_dataclass_field, dataclass_field_to_json_field, + field_to_env_var, ) from .constants import TAG from .decorators import try_with_load from .dumpers import get_dumper -from .enums import LetterCase, DateTimeTo +from .enums import DateTimeTo, LetterCase, LetterCasePriority +from .environ.loaders import EnvLoader from .errors import ParseError from .loaders import get_loader from .log import LOG from .models import Condition -from .type_def import E +from .type_def import E, EnvFileType from .utils.type_conv import date_to_timestamp, as_enum @@ -31,6 +33,45 @@ _debug_was_enabled = False +# use `debug_enabled` for log level if it's a str or int. +def _enable_debug_mode_if_needed(cls_loader, possible_lvl): + global _debug_was_enabled + if not _debug_was_enabled: + _debug_was_enabled = True + # use `debug_enabled` for log level if it's a str or int. + default_lvl = logging.DEBUG + # minimum logging level for logs by this library. + min_level = default_lvl if isinstance(possible_lvl, bool) else possible_lvl + # set the logging level of this library's logger. + LOG.setLevel(min_level) + LOG.info('DEBUG Mode is enabled') + + # Decorate all hooks so they format more helpful messages + # on error. + load_hooks = cls_loader.__LOAD_HOOKS__ + for typ in load_hooks: + load_hooks[typ] = try_with_load(load_hooks[typ]) + + +def _as_enum_safe(cls: type, name: str, base_type: Type[E]) -> Optional[E]: + """ + Attempt to return the value for class attribute :attr:`attr_name` as + a :type:`base_type`. + + :raises ParseError: If we are unable to convert the value of the class + attribute to an Enum of type `base_type`. + """ + try: + return as_enum(getattr(cls, name), base_type) + + except ParseError as e: + # We run into a parsing error while loading the enum; Add + # additional info on the Exception object before re-raising it + e.class_name = get_class_name(cls) + e.field_name = name + raise + + class BaseJSONWizardMeta(AbstractMeta): """ Superclass definition for the `JSONWizard.Meta` inner class. @@ -86,25 +127,9 @@ def bind_to(cls, dataclass: Type, create=True, is_default=True): cls_dumper = get_dumper(dataclass, create=create) if cls.debug_enabled: - global _debug_was_enabled - if not _debug_was_enabled: - _debug_was_enabled = True - # use `debug_enabled` for log level if it's a str or int. - possible_lvl = cls.debug_enabled - default_lvl = logging.DEBUG - # minimum logging level for logs by this library. - min_level = default_lvl if isinstance(possible_lvl, bool) else possible_lvl - # set the logging level of this library's logger. - LOG.setLevel(min_level) - LOG.info('DEBUG Mode is enabled') - - # Decorate all hooks so they format more helpful messages - # on error. - load_hooks = cls_loader.__LOAD_HOOKS__ - for typ in load_hooks: - load_hooks[typ] = try_with_load(load_hooks[typ]) - - if cls.json_key_to_field: + _enable_debug_mode_if_needed(cls_loader, cls.debug_enabled) + + if cls.json_key_to_field is not None: add_for_both = cls.json_key_to_field.pop('__all__', None) json_field_to_dataclass_field(dataclass).update( @@ -123,8 +148,8 @@ def bind_to(cls, dataclass: Type, create=True, is_default=True): if field not in dataclass_to_json_field: dataclass_to_json_field[field] = json_key - if cls.marshal_date_time_as: - enum_val = cls._as_enum_safe('marshal_date_time_as', DateTimeTo) + if cls.marshal_date_time_as is not None: + enum_val = _as_enum_safe(cls, 'marshal_date_time_as', DateTimeTo) if enum_val is DateTimeTo.TIMESTAMP: # Update dump hooks for the `datetime` and `date` types @@ -140,13 +165,13 @@ def bind_to(cls, dataclass: Type, create=True, is_default=True): # already serializes using this approach. pass - if cls.key_transform_with_load: - cls_loader.transform_json_field = cls._as_enum_safe( - 'key_transform_with_load', LetterCase) + if cls.key_transform_with_load is not None: + cls_loader.transform_json_field = _as_enum_safe( + cls, 'key_transform_with_load', LetterCase) - if cls.key_transform_with_dump: - cls_dumper.transform_dataclass_field = cls._as_enum_safe( - 'key_transform_with_dump', LetterCase) + if cls.key_transform_with_dump is not None: + cls_dumper.transform_dataclass_field = _as_enum_safe( + cls, 'key_transform_with_dump', LetterCase) # Finally, if needed, save the meta config for the outer class. This # will allow us to access this config as part of the JSON load/dump @@ -159,24 +184,72 @@ def bind_to(cls, dataclass: Type, create=True, is_default=True): else: _META[dataclass] = cls + +class BaseEnvWizardMeta(AbstractEnvMeta): + """ + Superclass definition for the `EnvWizard.Meta` inner class. + + See the implementation of the :class:`AbstractEnvMeta` class for the + available config that can be set, as well as for descriptions on any + implemented methods. + """ + + __slots__ = () + @classmethod - def _as_enum_safe(cls, name: str, base_type: Type[E]) -> Optional[E]: + def _init_subclass(cls): """ - Attempt to return the value for class attribute :attr:`attr_name` as - a :type:`base_type`. + Hook that should ideally be run whenever the `Meta` class is + sub-classed. - :raises ParseError: If we are unable to convert the value of the class - attribute to an Enum of type `base_type`. """ - try: - return as_enum(getattr(cls, name), base_type) + outer_cls_name = get_outer_class_name(cls, raise_=False) + + if outer_cls_name is not None: + META_INITIALIZER[outer_cls_name] = cls.bind_to + else: + # The `Meta` class is defined as an outer class. Emit a warning + # here, just so we can ensure awareness of this special case. + LOG.warning('The %r class is not declared as an Inner Class, so ' + 'these are global settings that will apply to all ' + 'EnvWizard sub-classes.', get_class_name(cls)) + + # Copy over global defaults to the :class:`AbstractMeta` + for attr in AbstractEnvMeta.fields_to_merge: + setattr(AbstractEnvMeta, attr, getattr(cls, attr, None)) + if cls.field_to_env_var: + AbstractEnvMeta.field_to_env_var = cls.field_to_env_var + + # Create a new class of `Type[W]`, and then pass `create=False` so + # that we don't create new loader / dumper for the class. + new_cls = create_new_class(cls, (AbstractJSONWizard, )) + cls.bind_to(new_cls, create=False) - except ParseError as e: - # We run into a parsing error while loading the enum; Add - # additional info on the Exception object before re-raising it - e.class_name = get_class_name(cls) - e.field_name = name - raise + @classmethod + def bind_to(cls, env_class: Type, create=True): + + cls_loader = get_loader(env_class, create=create, base_cls=EnvLoader) + cls_dumper = get_dumper(env_class, create=create) + + if cls.debug_enabled: + _enable_debug_mode_if_needed(cls_loader, cls.debug_enabled) + + if cls.field_to_env_var is not None: + field_to_env_var(env_class).update( + cls.field_to_env_var + ) + + cls.key_lookup_with_load = _as_enum_safe( + cls, 'key_lookup_with_load', LetterCasePriority) + + cls_dumper.transform_dataclass_field = _as_enum_safe( + cls, 'key_transform_with_dump', LetterCase) + + # Finally, if needed, save the meta config for the outer class. This + # will allow us to access this config as part of the JSON load/dump + # process if needed. + + _META[env_class] = cls # noinspection PyPep8Naming @@ -268,3 +341,44 @@ def DumpMeta(*, debug_enabled: 'bool | int | str' = False, # Create a new subclass of :class:`AbstractMeta` # noinspection PyTypeChecker return type('Meta', (BaseJSONWizardMeta, ), base_dict) + + +# noinspection PyPep8Naming +def EnvMeta(*, debug_enabled: 'bool | int | str' = False, + env_file: EnvFileType = None, + field_to_env_var: dict[str, str] = None, + key_lookup_with_load: Union[LetterCasePriority, str] = LetterCasePriority.SCREAMING_SNAKE, + key_transform_with_dump: Union[LetterCase, str] = LetterCase.SNAKE, + # marshal_date_time_as: Union[DateTimeTo, str] = None, + skip_defaults: bool = False, + # skip_if: Condition = None, + # skip_defaults_if: Condition = None, + ) -> META: + """ + Helper function to setup the ``Meta`` Config for the EnvWizard. + + For descriptions on what each of these params does, refer to the `Docs`_ + below, or check out the :class:`AbstractEnvMeta` definition (I want to avoid + duplicating the descriptions for params here). + + Examples:: + + >>> EnvMeta(key_transform_with_dump='SNAKE').bind_to(MyClass) + + .. _Docs: https://dataclass-wizard.readthedocs.io/en/latest/common_use_cases/meta.html + """ + + # Set meta attributes here. + base_dict = { + '__slots__': (), + 'debug_enabled': debug_enabled, + 'env_file': env_file, + 'field_to_env_var': field_to_env_var, + 'key_lookup_with_load': key_lookup_with_load, + 'key_transform_with_dump': key_transform_with_dump, + 'skip_defaults': skip_defaults, + } + + # Create a new subclass of :class:`AbstractMeta` + # noinspection PyTypeChecker + return type('Meta', (BaseEnvWizardMeta, ), base_dict) diff --git a/dataclass_wizard/class_helper.py b/dataclass_wizard/class_helper.py index eb8e0d8b..61f9f3f2 100644 --- a/dataclass_wizard/class_helper.py +++ b/dataclass_wizard/class_helper.py @@ -53,6 +53,9 @@ # A cached mapping, per dataclass, of instance field name to `SkipIf` condition DATACLASS_FIELD_TO_SKIP_IF = defaultdict(dict) +# A cached mapping, per `EnvWizard` subclass, of field name to env variable +FIELD_TO_ENV_VAR = defaultdict(dict) + # A mapping of dataclass name to its Meta initializer (defined in # :class:`bases.BaseJSONWizardMeta`), which is only set when the # :class:`JSONSerializable.Meta` is sub-classed. @@ -111,6 +114,13 @@ def dataclass_field_to_skip_if(cls): return DATACLASS_FIELD_TO_SKIP_IF[cls] +def field_to_env_var(cls): + """ + Returns a mapping of field in the `EnvWizard` subclass to env variable. + """ + return FIELD_TO_ENV_VAR[cls] + + def dataclass_field_to_load_parser( cls_loader, cls, @@ -286,16 +296,23 @@ def setup_dump_config_for_cls_if_needed(cls): def call_meta_initializer_if_needed(cls): - + """ + Calls the Meta initializer when the inner :class:`Meta` is sub-classed. + """ cls_name = get_class_name(cls) if cls_name in META_INITIALIZER: META_INITIALIZER[cls_name](cls) -def get_meta(cls): +def get_meta(cls, base_cls=AbstractMeta): + """ + Retrieves the Meta config for the :class:`AbstractJSONWizard` subclass. return _META.get(cls, AbstractMeta) + This config is set when the inner :class:`Meta` is sub-classed. + """ + return _META.get(cls, base_cls) def dataclass_fields(cls): diff --git a/dataclass_wizard/class_helper.pyi b/dataclass_wizard/class_helper.pyi index c403aa4b..e29a3959 100644 --- a/dataclass_wizard/class_helper.pyi +++ b/dataclass_wizard/class_helper.pyi @@ -2,8 +2,8 @@ from collections import defaultdict from dataclasses import Field from typing import Any, Callable -from .abstractions import W, AbstractLoader, AbstractDumper, AbstractParser -from .bases import META +from .abstractions import W, AbstractLoader, AbstractDumper, AbstractParser, E +from .bases import META, AbstractMeta from .models import Condition from .type_def import ExplicitNullType, T from .utils.dict_helper import DictWithLowerStore @@ -51,6 +51,9 @@ DATACLASS_FIELD_TO_JSON_FIELD: dict[type, dict[str, str]] = defaultdict(dict) # A cached mapping, per dataclass, of instance field name to `SkipIf` condition DATACLASS_FIELD_TO_SKIP_IF: dict[type, dict[str, Condition]] = defaultdict(dict) +# A cached mapping, per `EnvWizard` subclass, of field name to env variable +FIELD_TO_ENV_VAR: dict[type, dict[str, str]] = defaultdict(dict) + # A mapping of dataclass name to its Meta initializer (defined in # :class:`bases.BaseJSONWizardMeta`), which is only set when the # :class:`JSONSerializable.Meta` is sub-classed. @@ -109,6 +112,12 @@ def dataclass_field_to_skip_if(cls: type) -> dict[str, Condition]: """ +def field_to_env_var(cls: type) -> dict[str, str]: + """ + Returns a mapping of field in the `EnvWizard` subclass to env variable. + """ + + def dataclass_field_to_load_parser( cls_loader: type[AbstractLoader], cls: type, @@ -168,13 +177,13 @@ def setup_dump_config_for_cls_if_needed(cls: type) -> None: """ -def call_meta_initializer_if_needed(cls: type[W]) -> None: +def call_meta_initializer_if_needed(cls: type[W | E]) -> None: """ Calls the Meta initializer when the inner :class:`Meta` is sub-classed. """ -def get_meta(cls: type) -> META: +def get_meta(cls: type, base_cls: T = AbstractMeta) -> T | META: """ Retrieves the Meta config for the :class:`AbstractJSONWizard` subclass. diff --git a/dataclass_wizard/enums.py b/dataclass_wizard/enums.py index 803cbcd3..dc079ce5 100644 --- a/dataclass_wizard/enums.py +++ b/dataclass_wizard/enums.py @@ -4,6 +4,7 @@ """ from enum import Enum +from .environ import lookups from .utils.string_conv import * from .utils.wrappers import FuncWrapper @@ -33,3 +34,19 @@ class LetterCase(Enum): def __call__(self, *args): return self.value.f(*args) + + +class LetterCasePriority(Enum): + """ + Helper Enum which determines which letter casing we want to + *prioritize* when loading environment variable names. + + The default + """ + SCREAMING_SNAKE = FuncWrapper(lookups.with_screaming_snake_case) + SNAKE = FuncWrapper(lookups.with_snake_case) + CAMEL = FuncWrapper(lookups.with_pascal_or_camel_case) + PASCAL = FuncWrapper(lookups.with_pascal_or_camel_case) + + def __call__(self, *args): + return self.value.f(*args) diff --git a/dataclass_wizard/environ/__init__.py b/dataclass_wizard/environ/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dataclass_wizard/environ/dumpers.py b/dataclass_wizard/environ/dumpers.py new file mode 100644 index 00000000..689b87b6 --- /dev/null +++ b/dataclass_wizard/environ/dumpers.py @@ -0,0 +1,163 @@ + +from typing import List, Any, Optional, Callable, Dict, Type + +from .. import DumpMeta +from ..bases import META as M +from ..class_helper import ( + dataclass_field_to_default, + dataclass_field_to_json_field, + CLASS_TO_DUMP_FUNC, _META, +) +from ..dumpers import get_dumper, _asdict_inner +from ..enums import LetterCase +from ..type_def import ExplicitNull, JSONObject, T +from ..utils.string_conv import to_snake_case + + +def asdict(obj: T, + *, cls=None, dict_factory=dict, + exclude: List[str] = None, **kwargs) -> JSONObject: + """Return the fields of an instance of a `EnvWizard` subclass as a new + dictionary mapping field names to field values. + + Example usage:: + + class MyEnv(EnvWizard): + x: int + y: str + + env = MyEnv() + serialized = asdict(env) + + When directly invoking this function, an optional Meta configuration for + the `EnvWizard` subclass can be specified via ``DumpMeta``; by default, + this will apply recursively to any nested subclasses. Here's a sample + usage of this below:: + + >>> DumpMeta(key_transform='CAMEL').bind_to(MyClass) + >>> asdict(MyClass(my_str="value")) + + If given, 'dict_factory' will be used instead of built-in dict. + The function applies recursively to field values that are + `EnvWizard` subclasses. This will also look into built-in containers: + tuples, lists, and dicts. + """ + cls = cls or type(obj) + + try: + dump = CLASS_TO_DUMP_FUNC[cls] + except KeyError: + dump = dump_func_for_env_subclass(cls) + + return dump(obj, dict_factory, exclude, **kwargs) + + +def dump_func_for_env_subclass(cls: 'type[E]', + config: 'Optional[M]' = None, + nested_cls_to_dump_func: Dict[Type, Any] = None, + ) -> 'Callable[[E, Any, Any, Any], JSONObject]': + + # Get the dumper for the class, or create a new one as needed. + cls_dumper = get_dumper(cls) + + # Get the meta config for the class, or the default config otherwise. + # + # Only add the key transform if Meta config has not been specified + # for the `EnvWizard` subclass. + if cls in _META: + meta = _META[cls] + # TODO check if there a way to avoid this. The reason we are calling + # `DumpMeta` here is we have an `AbstractEnvMeta` type, which is not + # compatible with `AbstractMeta`. The `_asdict_inner` function calls + # `__or__` when it sees a nested dataclass type, which requires two + # `AbstractMeta` sub-types. + meta = DumpMeta(key_transform=meta.key_transform_with_dump, + skip_defaults=meta.skip_defaults) + + else: + # see the note above - converting to `DumpMeta` is not ideal. + meta = DumpMeta(key_transform=LetterCase.SNAKE) + cls_dumper.transform_dataclass_field = to_snake_case + + # we assume we're being run for the main dataclass (an `EnvWizard` subclass) + nested_cls_to_dump_func = {} + + # If the `recursive` flag is enabled and a Meta config is provided, + # apply the Meta recursively to any nested classes. + config = meta + + # This contains the dump hooks for the Env subclass. If the class + # sub-classes from `DumpMixIn`, these hooks could be customized. + hooks = cls_dumper.__DUMP_HOOKS__ + + # A cached mapping of each dataclass field to the resolved key name in a + # JSON or dictionary object; useful so we don't need to do a case + # transformation (via regex) each time. + env_subclass_to_json_field = dataclass_field_to_json_field(cls) + + # A cached mapping of dataclass field name to its default value, either + # via a `default` or `default_factory` argument. + field_to_default = dataclass_field_to_default(cls) + + # A collection of field names in the dataclass. + field_names = cls.__fields__.keys() + + def cls_asdict(obj: T, dict_factory=dict, + exclude: List[str] = None, + skip_defaults=meta.skip_defaults) -> JSONObject: + """ + Serialize an `EnvWizard` subclass `cls` to a Python dictionary object. + """ + + # Call the optional hook that runs before we process the subclass + cls_dumper.__pre_as_dict__(obj) + + # This a list that contains a mapping of each `EnvWizard` field to its + # serialized value. + result = [] + + # Loop over the `EnvWizard` fields + for field in field_names: + + # Get the resolved JSON field name + try: + json_field = env_subclass_to_json_field[field] + + except KeyError: + # Normalize the Env field name (by default to camel + # case) + json_field = cls_dumper.transform_dataclass_field(field) + env_subclass_to_json_field[field] = json_field + + # Exclude any fields that are explicitly ignored. + if json_field is ExplicitNull: + continue + if exclude and field in exclude: + continue + + # -- This line is *mostly* the same as in the original version -- + fv = getattr(obj, field) + + # Check if we need to strip defaults, and the field currently + # is assigned a default value. + # + # TODO: maybe it makes sense to move this logic to a separate + # function, as it might be slightly more performant. + if skip_defaults and field in field_to_default \ + and fv == field_to_default[field]: + continue + + value = _asdict_inner(fv, dict_factory, hooks, config, + nested_cls_to_dump_func) + + # -- This line is *mostly* the same as in the original version -- + result.append((json_field, value)) + + # -- This line is the same as in the original version -- + return dict_factory(result) + + # In any case, save the dump function for the class, so we don't need to + # run this logic each time. + CLASS_TO_DUMP_FUNC[cls] = cls_asdict + + return cls_asdict diff --git a/dataclass_wizard/environ/loaders.py b/dataclass_wizard/environ/loaders.py new file mode 100644 index 00000000..4196ced7 --- /dev/null +++ b/dataclass_wizard/environ/loaders.py @@ -0,0 +1,169 @@ +from datetime import datetime, date, timezone +from typing import ( + Type, Dict, List, Tuple, Iterable, Sequence, + Union, AnyStr, Optional, Callable, +) + +from ..abstractions import AbstractParser +from ..bases import META +from ..decorators import _single_arg_alias +from ..loaders import LoadMixin, load_func_for_dataclass +from ..type_def import ( + FrozenKeys, DefFactory, M, N, U, DD, LSQ, NT, T, JSONObject +) +from ..utils.type_conv import ( + as_datetime, as_date, as_list, as_dict +) + + +class EnvLoader(LoadMixin): + """ + This Mixin class derives its name from the eponymous `json.loads` + function. Essentially it contains helper methods to convert JSON strings + (or a Python dictionary object) to a `dataclass` which can often contain + complex types such as lists, dicts, or even other dataclasses nested + within it. + + Refer to the :class:`AbstractLoader` class for documentation on any of the + implemented methods. + + """ + __slots__ = () + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__() + + cls.register_load_hook(bytes, cls.load_to_bytes) + cls.register_load_hook(bytearray, cls.load_to_byte_array) + + @staticmethod + def load_to_bytes( + o: AnyStr, base_type: Type[bytes], encoding='utf-8') -> bytes: + + return base_type(o, encoding) + + @staticmethod + def load_to_byte_array( + o: AnyStr, base_type: Type[bytearray], + encoding='utf-8') -> bytearray: + + return base_type(o, encoding) if isinstance(o, str) else base_type(o) + + @staticmethod + @_single_arg_alias('base_type') + def load_to_uuid(o: Union[AnyStr, U], base_type: Type[U]) -> U: + # alias: base_type(o) + ... + + @staticmethod + def load_to_iterable( + o: Iterable, base_type: Type[LSQ], + elem_parser: AbstractParser) -> LSQ: + + return super(EnvLoader, EnvLoader).load_to_iterable( + as_list(o), base_type, elem_parser) + + @staticmethod + def load_to_tuple( + o: Union[List, Tuple], base_type: Type[Tuple], + elem_parsers: Sequence[AbstractParser]) -> Tuple: + + return super(EnvLoader, EnvLoader).load_to_tuple( + as_list(o), base_type, elem_parsers) + + @staticmethod + def load_to_named_tuple( + o: Union[Dict, List, Tuple], base_type: Type[NT], + field_to_parser: 'FieldToParser', + field_parsers: List[AbstractParser]) -> NT: + + # TODO check for both list and dict + + return super(EnvLoader, EnvLoader).load_to_named_tuple( + as_list(o), base_type, field_to_parser, field_parsers) + + @staticmethod + def load_to_named_tuple_untyped( + o: Union[Dict, List, Tuple], base_type: Type[NT], + dict_parser: AbstractParser, list_parser: AbstractParser) -> NT: + + return super(EnvLoader, EnvLoader).load_to_named_tuple_untyped( + as_list(o), base_type, dict_parser, list_parser) + + @staticmethod + def load_to_dict( + o: Dict, base_type: Type[M], + key_parser: AbstractParser, + val_parser: AbstractParser) -> M: + + return super(EnvLoader, EnvLoader).load_to_dict( + as_dict(o), base_type, key_parser, val_parser) + + @staticmethod + def load_to_defaultdict( + o: Dict, base_type: Type[DD], + default_factory: DefFactory, + key_parser: AbstractParser, + val_parser: AbstractParser) -> DD: + + return super(EnvLoader, EnvLoader).load_to_defaultdict( + as_dict(o), base_type, default_factory, key_parser, val_parser) + + @staticmethod + def load_to_typed_dict( + o: Dict, base_type: Type[M], + key_to_parser: 'FieldToParser', + required_keys: FrozenKeys, + optional_keys: FrozenKeys) -> M: + + return super(EnvLoader, EnvLoader).load_to_typed_dict( + as_dict(o), base_type, key_to_parser, required_keys, optional_keys) + + @staticmethod + def load_to_datetime( + o: Union[str, N], base_type: Type[datetime]) -> datetime: + if isinstance(o, str): + # Check if it's a string in numeric format, like '1.23' + if o.replace('.', '', 1).isdigit(): + return base_type.fromtimestamp(float(o), tz=timezone.utc) + + return base_type.fromisoformat(o.replace('Z', '+00:00', 1)) + + # default: as_datetime + return as_datetime(o, base_type) + + @staticmethod + def load_to_date(o: Union[str, N], base_type: Type[date]) -> date: + if isinstance(o, str): + # Check if it's a string in numeric format, like '1.23' + if o.replace('.', '', 1).isdigit(): + return base_type.fromtimestamp(float(o)) + + return base_type.fromisoformat(o) + + # default: as_date + return as_date(o, base_type) + + @staticmethod + def load_func_for_dataclass( + cls: Type[T], + config: Optional[META], + is_main_class: bool = False, + ) -> Callable[[JSONObject], T]: + + load = load_func_for_dataclass( + cls, + is_main_class=False, + config=config, + # override the loader class + loader_cls=EnvLoader, + ) + + def load_to_dataclass(o: 'str | JSONObject', *_): + """ + Receives either a string or a `dict` as an input, and return a + dataclass instance of type `cls`. + """ + return load(as_dict(o)) + + return load_to_dataclass diff --git a/dataclass_wizard/environ/lookups.py b/dataclass_wizard/environ/lookups.py new file mode 100644 index 00000000..8a116b1a --- /dev/null +++ b/dataclass_wizard/environ/lookups.py @@ -0,0 +1,232 @@ +import os +from dataclasses import MISSING +from os import environ, name +from typing import ClassVar, Dict, Optional, Set + +from ..decorators import cached_class_property +from ..lazy_imports import dotenv +from ..type_def import StrCollection, EnvFileType +from ..utils.string_conv import to_snake_case + + +# Type of `os.environ` or `DotEnv` dict +Environ = Dict[str, Optional[str]] + +# Type of (unique) environment variable names +EnvVars = Set[str] + + +# noinspection PyMethodParameters +class Env: + + __slots__ = () + + _accessed_cleaned_to_env: ClassVar[bool] = False + + @cached_class_property + def var_names(cls) -> EnvVars: + """ + Cached mapping of `os.environ` key names. This can be refreshed with + :meth:`reload` as needed. + """ + return set(environ) + + @classmethod + def reload(cls, env: dict = environ): + """Refresh cached environment variable names.""" + env_vars: EnvVars = cls.var_names + new_vars = set(env) - env_vars + + # update names of environment variables + env_vars.update(new_vars) + + # update mapping of cleaned environment variables (if needed) + if cls._accessed_cleaned_to_env: + cls.cleaned_to_env.update( + (clean(var), var) for var in new_vars + ) + + @classmethod + def dotenv_values(cls, files: EnvFileType) -> Environ: + """ + Retrieve the values (environment variables) from a dotenv file, + or a list/tuple of dotenv files. + """ + if isinstance(files, (str, os.PathLike)): + files = [files] + elif files is True: + files = ['.env'] + + env: Environ = {} + + for f in files: + # iterate backwards (from current directory) to find the + # dotenv file + dotenv_path = dotenv.find_dotenv(f) + # take environment variables from `.env` file + dotenv_values = dotenv.dotenv_values(dotenv_path) + env.update(dotenv_values) + + return env + + @classmethod + def update_with_dotenv(cls, files: EnvFileType = '.env', dotenv_values=None): + if dotenv_values is None: + dotenv_values = cls.dotenv_values(files) + + # reload cached mapping of environment variables + cls.reload(dotenv_values) + # update `os.environ` with new environment variables + environ.update(dotenv_values) + + # noinspection PyDunderSlots,PyUnresolvedReferences + @cached_class_property + def cleaned_to_env(cls) -> Environ: + cls._accessed_cleaned_to_env = True + return {clean(var): var for var in cls.var_names} + + +def clean(s: str) -> str: + """ + TODO: + see https://stackoverflow.com/questions/1276764/stripping-everything-but-alphanumeric-chars-from-a-string-in-python + also, see if we can refactor to use something like Rust and `pyo3` for a slight performance improvement. + """ + return s.replace('-', '').replace('_', '').lower() + + +def try_cleaned(key: str): + """ + Return the value of the env variable as a *string* if present in + the Environment, or `MISSING` otherwise. + """ + key = Env.cleaned_to_env.get(clean(key)) + + if key is not None: + return environ[key] + + return MISSING + + +if name == 'nt': + # Where Env Var Names Must Be UPPERCASE + def lookup_exact(var: StrCollection): + """ + Lookup by variable name(s) with *exact* letter casing, and return + `None` if not found in the environment. + """ + if isinstance(var, str): + var = var.upper() + + if var in Env.var_names: + return environ[var] + + else: # a collection of env variable names. + for v in var: + v = v.upper() + + if v in Env.var_names: + return environ[v] + + return MISSING + +else: + # Where Env Var Names Can Be Mixed Case + def lookup_exact(var: StrCollection): + """ + Lookup by variable name(s) with *exact* letter casing, and return + `None` if not found in the environment. + """ + if isinstance(var, str): + if var in Env.var_names: + return environ[var] + + else: # a collection of env variable names. + for v in var: + if v in Env.var_names: + return environ[v] + + return MISSING + + +def with_screaming_snake_case(field_name: str) -> Optional[str]: + """ + Lookup with `SCREAMING_SNAKE_CASE` letter casing first - this is the + default lookup. + + This function assumes the dataclass field name is lower-cased. + + For a field named 'my_env_var', this tries the following lookups in order: + - MY_ENV_VAR (screaming snake-case) + - my_env_var (snake-case) + - Any other variations - i.e. MyEnvVar, myEnvVar, myenvvar, my-env-var + + :param field_name: The dataclass field name to lookup in the environment. + :return: The value of the matched environment variable, if one is found in + the environment. + """ + upper_key = field_name.upper() + + if upper_key in Env.var_names: + return environ[upper_key] + + if field_name in Env.var_names: + return environ[field_name] + + return try_cleaned(field_name) + + +def with_snake_case(field_name: str) -> Optional[str]: + """Lookup with `snake_case` letter casing first. + + This function assumes the dataclass field name is lower-cased. + + For a field named 'my_env_var', this tries the following lookups in order: + - my_env_var (snake-case) + - MY_ENV_VAR (screaming snake-case) + - Any other variations - i.e. MyEnvVar, myEnvVar, myenvvar, my-env-var + + :param field_name: The dataclass field name to lookup in the environment. + :return: The value of the matched environment variable, if one is found in + the environment. + """ + if field_name in Env.var_names: + return environ[field_name] + + upper_key = field_name.upper() + + if upper_key in Env.var_names: + return environ[upper_key] + + return try_cleaned(field_name) + + +def with_pascal_or_camel_case(field_name: str) -> Optional[str]: + """Lookup with `PascalCase` or `camelCase` letter casing first. + + This function assumes the dataclass field name is either pascal- or camel- + cased. + + For a field named 'myEnvVar', this tries the following lookups in order: + - myEnvVar, MyEnvVar (camel-case, or pascal-case) + - MY_ENV_VAR (screaming snake-case) + - my_env_var (snake-case) + - Any other variations - i.e. my-env-var, myenvvar + + :param field_name: The dataclass field name to lookup in the environment. + :return: The value of the matched environment variable, if one is found in + the environment. + """ + if field_name in Env.var_names: + return environ[field_name] + + snake_key = to_snake_case(field_name) + upper_key = snake_key.upper() + + if upper_key in Env.var_names: + return environ[upper_key] + + if snake_key in Env.var_names: + return environ[snake_key] + + return try_cleaned(field_name) diff --git a/dataclass_wizard/environ/wizard.py b/dataclass_wizard/environ/wizard.py new file mode 100644 index 00000000..f92f36ce --- /dev/null +++ b/dataclass_wizard/environ/wizard.py @@ -0,0 +1,266 @@ +import json +import logging +from dataclasses import MISSING, dataclass, fields, Field +from typing import AnyStr, Callable, dataclass_transform + +from .dumpers import asdict +from .lookups import Env, lookup_exact, clean +from ..abstractions import AbstractEnvWizard +from ..bases import AbstractEnvMeta +from ..bases_meta import BaseEnvWizardMeta, LoadMeta, EnvMeta +from ..class_helper import (call_meta_initializer_if_needed, get_meta, + field_to_env_var, dataclass_field_to_json_field) +from ..decorators import cached_class_property, _alias +from ..environ.loaders import EnvLoader +from ..errors import ExtraData, MissingVars, ParseError, type_name +from ..loaders import get_loader +from ..models import Extras, JSONField +from ..type_def import JSONObject, Encoder, EnvFileType, ExplicitNull +from ..utils.function_builder import FunctionBuilder + +_to_dataclass = dataclass(init=False) + + +@dataclass_transform(kw_only_default=True) +class EnvWizard(AbstractEnvWizard): + __slots__ = () + + class Meta(BaseEnvWizardMeta): + """ + Inner meta class that can be extended by sub-classes for additional + customization with the environment load process. + """ + __slots__ = () + + # Class attribute to enable detection of the class type. + __is_inner_meta__ = True + + def __init_subclass__(cls): + # Set the `__init_subclass__` method here, so we can ensure it + # doesn't run for the `EnvWizard.Meta` class. + return cls._init_subclass() + + # noinspection PyMethodParameters + @cached_class_property + def __fields__(cls: 'E') -> 'dict[str, Field]': + cls_fields = {} + field_to_var = field_to_env_var(cls) + + for field in fields(cls): + name = field.name + cls_fields[name] = field + + if isinstance(field, JSONField): + if not field.json.dump: + field_to_json_key = dataclass_field_to_json_field(cls) + field_to_json_key[name] = ExplicitNull + + keys = field.json.keys + if keys: + # minor optimization: convert a one-element tuple of `str` to `str` + field_to_var[name] = keys[0] if len(keys) == 1 else keys + + return cls_fields + + @_alias(asdict) + def to_dict(self: 'E', exclude: 'list[str]' = None, + skip_defaults: bool = False) -> JSONObject: + """ + Converts the `EnvWizard` subclass to a Python dictionary object that + is JSON serializable. + """ + # alias: asdict(self) + ... + + def to_json(self: 'E', *, + encoder: Encoder = json.dumps, + **encoder_kwargs) -> AnyStr: + """ + Converts the `EnvWizard` subclass to a JSON `string` representation. + """ + return encoder(asdict(self), **encoder_kwargs) + + # # stub for type hinting purposes. + # def __init__(self, *, + # _env_file: EnvFileType = None, + # _reload_env: bool = False, + # **init_kwargs) -> None: + # ... + + def __init_subclass__(cls, *, reload_env=False, debug=False): + + if reload_env: # reload cached var names from `os.environ` as needed. + Env.reload() + + # apply the `@dataclass(init=False)` decorator to `cls`. + _to_dataclass(cls) + + if debug: + default_lvl = logging.DEBUG + logging.basicConfig(level=default_lvl) + # minimum logging level for logs by this library + min_level = default_lvl if isinstance(debug, bool) else debug + # set `debug_enabled` flag for the class's Meta + EnvMeta(debug_enabled=min_level).bind_to(cls) + + # Calls the Meta initializer when inner :class:`Meta` is sub-classed. + call_meta_initializer_if_needed(cls) + + # create and set the `__init__()` method. + cls.__init__ = cls._init_fn() + + @classmethod + def _init_fn(cls) -> Callable: + """ + Returns a generated ``__init__()`` constructor method for the + :class:`EnvWizard` subclass, vis-à-vis how the ``dataclasses`` + module does it, with a few noticeable differences. + """ + + meta = get_meta(cls, base_cls=AbstractEnvMeta) + cls_loader = get_loader(cls, base_cls=EnvLoader) + + # A cached mapping of each dataclass field name to its environment + # variable name; useful so we don't need to do a case transformation + # (via regex) each time. + field_to_var = field_to_env_var(cls) + + # The function to case-transform and lookup variables defined in the + # environment. + get_env: 'Callable[[str], str | None]' = meta.key_lookup_with_load + + # noinspection PyArgumentList + extras = Extras(config=None) + + cls_fields: 'dict[str, Field]' = cls.__fields__ + field_names = frozenset(cls_fields) + + _meta_env_file = meta.env_file + + _locals = {'Env': Env, + 'EnvFileType': EnvFileType, + 'MISSING': MISSING, + 'ParseError': ParseError, + 'field_names': field_names, + 'get_env': get_env, + 'lookup_exact': lookup_exact} + + _globals = {'MissingVars': MissingVars, + 'add': _add_missing_var, + 'cls': cls, + 'fields_ordered': cls_fields.keys(), + 'handle_err': _handle_parse_error} + + # parameters to the `__init__()` method. + init_params = ['self', + 'env_file:EnvFileType=None', + 'reload_env:bool=False'] + + fn_gen = FunctionBuilder() + + with fn_gen.function('__init__', init_params, None): + # reload cached var names from `os.environ` as needed. + with fn_gen.if_('reload_env'): + fn_gen.add_line('Env.reload()') + # update environment with values in the "dot env" files as needed. + if _meta_env_file: + fn = fn_gen.elif_ + _globals['_dotenv_values'] = Env.dotenv_values(_meta_env_file) + with fn_gen.if_('env_file is None'): + fn_gen.add_line('Env.update_with_dotenv(dotenv_values=_dotenv_values)') + else: + fn = fn_gen.if_ + with fn('env_file'): + fn_gen.add_line('Env.update_with_dotenv(env_file)') + + # iterate over the dataclass fields and (attempt to) resolve + # each one. + fn_gen.add_line('missing_vars = []') + + for name, f in cls_fields.items(): + type_field = f'_tp_{name}' + tp = _globals[type_field] = f.type + + init_params.append(f'{name}:{type_field}=MISSING') + + # retrieve value (if it exists) for the environment variable + + env_var = field_to_var.get(name) + if env_var: + part = f'({name} := lookup_exact({env_var!r}))' + else: + part = f'({name} := get_env({name!r}))' + + with fn_gen.if_(f'{name} is not MISSING or {part} is not MISSING'): + parser_name = f'_parser_{name}' + _globals[parser_name] = cls_loader.get_parser_for_annotation( + tp, cls, extras) + with fn_gen.try_(): + fn_gen.add_line(f'self.{name} = {parser_name}({name})') + with fn_gen.except_(ParseError, 'e'): + fn_gen.add_line(f'handle_err(e, cls, {name!r}, {env_var!r})') + # this `else` block means that a value was not received for the + # field, either via keyword arguments or Environment. + with fn_gen.else_(): + # check if the field defines a `default` or `default_factory` + # value; note this is similar to how `dataclasses` does it. + default_name = f'_dflt_{name}' + if f.default is not MISSING: + _globals[default_name] = f.default + fn_gen.add_line(f'self.{name} = {default_name}') + elif f.default_factory is not MISSING: + _globals[default_name] = f.default_factory + fn_gen.add_line(f'self.{name} = {default_name}()') + else: + fn_gen.add_line(f'add(missing_vars, {name!r}, {type_field})') + + # check for any required fields with missing values + with fn_gen.if_('missing_vars'): + fn_gen.add_line('raise MissingVars(cls, missing_vars) from None') + + # if keyword arguments are passed in, confirm that all there + # aren't any "extra" keyword arguments + # if _extra is not Extra.IGNORE: + # with fn_gen.if_('has_kwargs'): + # # get a list of keyword arguments that don't map to any fields + # fn_gen.add_line('extra_kwargs = set(init_kwargs) - field_names') + # with fn_gen.if_('extra_kwargs'): + # # the default behavior is "DENY", so an error will be raised here. + # if _extra is None or _extra is Extra.DENY: + # _globals['ExtraData'] = ExtraData + # fn_gen.add_line('raise ExtraData(cls, extra_kwargs, list(fields_ordered)) from None') + # else: # Extra.ALLOW + # # else, if we want to "ALLOW" extra keyword arguments, we need to + # # store those attributes in the instance. + # with fn_gen.for_('attr in extra_kwargs'): + # fn_gen.add_line('setattr(self, attr, init_kwargs[attr])') + + functions = fn_gen.create_functions(globals=_globals, locals=_locals) + + return functions['__init__'] + + +def _add_missing_var(missing_vars: list, name: str, tp: type): + # noinspection PyBroadException + try: + suggested = tp() + except Exception: + suggested = None + tn = type_name(tp) + missing_vars.append((name, tn, suggested)) + + +def _handle_parse_error(e: ParseError, + cls: type, name: str, + var_name: 'str | None'): + + # We run into a parsing error while loading the field + # value; Add additional info on the Exception object + # before re-raising it. + e.class_name = cls + e.field_name = name + if var_name is None: + var_name = Env.cleaned_to_env.get(clean(name), name) + e.kwargs['env_variable'] = var_name + + raise diff --git a/dataclass_wizard/errors.py b/dataclass_wizard/errors.py index 461fdd79..698e5a32 100644 --- a/dataclass_wizard/errors.py +++ b/dataclass_wizard/errors.py @@ -1,8 +1,7 @@ -import json from abc import ABC, abstractmethod from dataclasses import Field, MISSING from typing import (Any, Type, Dict, Tuple, ClassVar, - Optional, Union, Iterable, Callable) + Optional, Union, Iterable, Callable, Collection, Sequence) from .utils.string_conv import normalize @@ -11,6 +10,19 @@ JSONObject = Dict[str, Any] +def type_name(obj: type) -> str: + """Return the type or class name of an object""" + from .utils.typing_compat import is_generic + + # for type generics like `dict[str, float]`, we want to return + # the subscripted value as is, rather than simply accessing the + # `__name__` property, which in this case would be `dict` instead. + if is_generic(obj): + return str(obj) + + return getattr(obj, '__qualname__', getattr(obj, '__name__', repr(obj))) + + def show_deprecation_warning( fn: Callable, reason: str, @@ -134,6 +146,8 @@ def message(self) -> str: if self.json_object: self.kwargs['json_object'] = json.dumps(self.json_object, default=str) + from .utils.json_util import safe_dumps + self.kwargs['json_object'] = safe_dumps(self.json_object) if self.kwargs: sep = '\n ' @@ -143,6 +157,44 @@ def message(self) -> str: return msg +class ExtraData(JSONWizardError): + """ + Error raised when extra keyword arguments are passed in to the constructor + or `__init__()` method of an `EnvWizard` subclass. + + Note that this error class is raised by default, unless a value for the + `extra` field is specified in the :class:`Meta` class. + """ + + _TEMPLATE = ('{cls}.__init__() received extra keyword arguments:\n' + ' extras: {extra_kwargs!r}\n' + ' fields: {field_names!r}\n' + ' resolution: specify a value for `extra` in the Meta ' + 'config for the class, to control how extra keyword ' + 'arguments are handled.') + + def __init__(self, + cls: Type, + extra_kwargs: Collection[str], + field_names: Collection[str]): + + super().__init__() + + self.class_name: str = type_name(cls) + self.extra_kwargs = extra_kwargs + self.field_names = field_names + + @property + def message(self) -> str: + msg = self._TEMPLATE.format( + cls=self.class_name, + extra_kwargs=self.extra_kwargs, + field_names=self.field_names, + ) + + return msg + + class MissingFields(JSONWizardError): """ Error raised when unable to create a class instance (most likely due to @@ -193,9 +245,11 @@ def __init__(self, base_err: Exception, @property def message(self) -> str: + from .utils.json_util import safe_dumps + msg = self._TEMPLATE.format( cls=self.class_name, - json_string=json.dumps(self.obj, default=str), + json_string=safe_dumps(self.obj), e=self.base_error, fields=self.fields, missing_fields=self.missing_fields) @@ -235,11 +289,16 @@ def __init__(self, self.kwargs = kwargs self.class_name: str = self.name(cls) + # self.class_name: str = type_name(cls) + @property def message(self) -> str: + from .utils.json_util import safe_dumps + msg = self._TEMPLATE.format( cls=self.class_name, - json_string=json.dumps(self.obj, default=str), + # json_string=json.dumps(self.obj, default=str), + json_string=safe_dumps(self.obj), fields=self.fields, json_key=self.json_key) @@ -267,12 +326,17 @@ def __init__(self, nested_cls: Type, **kwargs): super().__init__(self, None, nested_cls, **kwargs) self.nested_class_name: str = self.name(nested_cls) + # self.nested_class_name: str = type_name(nested_cls) + @property def message(self) -> str: + from .utils.json_util import safe_dumps + msg = self._TEMPLATE.format( cls=self.class_name, nested_cls=self.nested_class_name, - json_string=json.dumps(self.obj, default=str), + # json_string=json.dumps(self.obj, default=str), + json_string=safe_dumps(self.obj), field=self.field_name, o=self.obj, ) @@ -329,3 +393,53 @@ def __init__(self, cls: Type, field_name: str): def message(self) -> str: return self._TEMPLATE.format(cls=self.class_name, field=self.field_name) + + +class MissingVars(JSONWizardError): + """ + Error raised when unable to create an instance of a EnvWizard subclass + (most likely due to missing environment variables in the Environment) + + """ + _TEMPLATE = ('{prefix} in class `{cls}` missing in the Environment:\n' + '{fields}\n\n' + 'resolution #1: set a default value for any optional fields, as below.\n\n' + '{def_resolution}' + '\n\n...\n' + 'resolution #2: pass in values for required fields to {cls}.__init__():\n\n' + ' {init_resolution}') + + def __init__(self, + cls: Type, + missing_vars: Sequence[Tuple[str, str, Any]]): + + super().__init__() + + indent = ' ' * 4 + + self.class_name: str = type_name(cls) + self.fields = '\n'.join([f'{indent}- {f[0]}' for f in missing_vars]) + self.def_resolution = '\n'.join([f'class {self.class_name}:'] + + [f'{indent}{f}: {typ} = {default!r}' + for (f, typ, default) in missing_vars]) + + init_vars = ', '.join([f'{f}={default!r}' for (f, typ, default) in missing_vars]) + self.init_resolution = f'instance = {self.class_name}({init_vars})' + + num_fields = len(missing_vars) + if num_fields > 1: + self.prefix = f'There are {len(missing_vars)} required fields' + else: + self.prefix = f'There is {len(missing_vars)} required field' + + @property + def message(self) -> str: + msg = self._TEMPLATE.format( + cls=self.class_name, + prefix=self.prefix, + fields=self.fields, + def_resolution=self.def_resolution, + init_resolution=self.init_resolution, + ) + + return msg diff --git a/dataclass_wizard/lazy_imports.py b/dataclass_wizard/lazy_imports.py index 058a96d9..f808a076 100644 --- a/dataclass_wizard/lazy_imports.py +++ b/dataclass_wizard/lazy_imports.py @@ -9,6 +9,9 @@ from .utils.lazy_loader import LazyLoader +# python-dotenv: for loading environment values from `.env` files +dotenv = LazyLoader(globals(), 'dotenv', 'dotenv', local_name='python-dotenv') + # pytimeparse: for parsing JSON string values as a `datetime.timedelta` pytimeparse = LazyLoader(globals(), 'pytimeparse', 'timedelta') diff --git a/dataclass_wizard/loaders.py b/dataclass_wizard/loaders.py index 39111fea..50167247 100644 --- a/dataclass_wizard/loaders.py +++ b/dataclass_wizard/loaders.py @@ -12,7 +12,7 @@ ) from uuid import UUID -from .abstractions import AbstractLoader, AbstractParser, FieldToParser +from .abstractions import AbstractLoader, AbstractParser from .bases import BaseLoadHook, AbstractMeta, META from .class_helper import ( create_new_class, @@ -144,7 +144,7 @@ def load_to_tuple( @staticmethod def load_to_named_tuple( o: Union[Dict, List, Tuple], base_type: Type[NT], - field_to_parser: FieldToParser, + field_to_parser: 'FieldToParser', field_parsers: List[AbstractParser]) -> NT: if isinstance(o, dict): @@ -195,7 +195,7 @@ def load_to_defaultdict( @staticmethod def load_to_typed_dict( o: Dict, base_type: Type[M], - key_to_parser: FieldToParser, + key_to_parser: 'FieldToParser', required_keys: FrozenKeys, optional_keys: FrozenKeys) -> M: @@ -248,6 +248,15 @@ def load_to_timedelta( # alias: as_timedelta ... + @staticmethod + def load_func_for_dataclass( + cls: Type[T], + config: Optional[META], + ) -> Callable[[JSONObject], T]: + + return load_func_for_dataclass( + cls, is_main_class=False, config=config) + @classmethod def get_parser_for_annotation(cls, ann_type: Type[T], base_cls: Type = None, @@ -520,7 +529,8 @@ def setup_default_loader(cls=LoadMixin): cls.register_load_hook(timedelta, cls.load_to_timedelta) -def get_loader(class_or_instance=None, create=True) -> Type[LoadMixin]: +def get_loader(class_or_instance=None, create=True, + base_cls: T = LoadMixin) -> Type[T]: """ Get the loader for the class, using the following logic: @@ -541,10 +551,10 @@ def get_loader(class_or_instance=None, create=True) -> Type[LoadMixin]: return set_class_loader(class_or_instance, class_or_instance) elif create: - cls_loader = create_new_class(class_or_instance, (LoadMixin, )) + cls_loader = create_new_class(class_or_instance, (base_cls, )) return set_class_loader(class_or_instance, cls_loader) - return set_class_loader(class_or_instance, LoadMixin) + return set_class_loader(class_or_instance, base_cls) def fromdict(cls: Type[T], d: JSONObject) -> T: @@ -591,6 +601,7 @@ def load_func_for_dataclass( cls: Type[T], is_main_class: bool = True, config: Optional[META] = None, + loader_cls=LoadMixin, ) -> Callable[[JSONObject], T]: # TODO dynamically generate for multiple nested classes at once @@ -598,8 +609,9 @@ def load_func_for_dataclass( # Tuple describing the fields of this dataclass. cls_fields = dataclass_fields(cls) + # Get the loader for the class, or create a new one as needed. - cls_loader = get_loader(cls) + cls_loader = get_loader(cls, base_cls=loader_cls) # Get the meta config for the class, or the default config otherwise. meta = get_meta(cls) @@ -670,7 +682,7 @@ def load_func_for_dataclass( if has_json_paths: loop_over_o = num_paths != len(dataclass_init_fields(cls)) - _locals['get_safe'] = safe_get + _locals['safe_get'] = safe_get else: loop_over_o = True @@ -698,7 +710,7 @@ def load_func_for_dataclass( extra_args = f', {default_value}' else: extra_args = '' - fn_gen.add_line(f'field={field!r}; init_kwargs[field] = field_to_parser[field](get_safe(o, {path!r}{extra_args}))') + fn_gen.add_line(f'field={field!r}; init_kwargs[field] = field_to_parser[field](safe_get(o, {path!r}{extra_args}))') with fn_gen.except_(ParseError, 'e'): # We run into a parsing error while loading the field value; diff --git a/dataclass_wizard/models.py b/dataclass_wizard/models.py index d23948a2..60bc0b60 100644 --- a/dataclass_wizard/models.py +++ b/dataclass_wizard/models.py @@ -5,10 +5,10 @@ from .constants import PY310_OR_ABOVE from .decorators import cached_property -from .type_def import T, DT, PyTypedDict # noinspection PyProtectedMember from .utils.dataclass_compat import _create_fn from .utils.object_path import split_object_path +from .type_def import T, DT, PyTypedDict from .utils.type_conv import as_datetime, as_time, as_date @@ -22,6 +22,8 @@ # if PY312_OR_ABOVE: # type CatchAll = Mapping CatchAll = NewType('CatchAll', Mapping) +# A date, time, datetime sub type, or None. +# DT_OR_NONE = Optional[DT] class Extras(PyTypedDict): @@ -94,6 +96,9 @@ def __init__(self, keys, all: bool, dump: bool, if isinstance(keys, str): keys = split_object_path(keys) if path else (keys,) + # keys = (keys, ) + # elif keys is Ellipsis: + # keys = () self.json = JSON(*keys, all=all, dump=dump, path=path) @@ -108,6 +113,9 @@ def __init__(self, keys, all: bool, dump: bool, if isinstance(keys, str): keys = split_object_path(keys) if path else (keys,) + # keys = (keys, ) + # elif keys is Ellipsis: + # keys = () self.json = JSON(*keys, all=all, dump=dump, path=path) diff --git a/dataclass_wizard/parsers.py b/dataclass_wizard/parsers.py index 55a32f47..8d786e84 100644 --- a/dataclass_wizard/parsers.py +++ b/dataclass_wizard/parsers.py @@ -20,7 +20,7 @@ Type, Any, Optional, Tuple, Dict, Iterable, Callable, List ) -from .abstractions import AbstractParser, FieldToParser +from .abstractions import AbstractParser from .bases import AbstractMeta from .class_helper import get_meta, _META from .constants import TAG @@ -345,17 +345,7 @@ def __call__(self, o: Iterable) -> LSQ: See the declaration of :var:`LSQ` for more info. """ - try: - return self.hook(o, self.base_type, self.elem_parser) - # TODO - except Exception: - if not isinstance(o, self.base_type): - e = TypeError('Incorrect type for field') - raise ParseError( - e, o, self.base_type, - desired_type=self.base_type) - else: - raise + return self.hook(o, self.base_type, self.elem_parser) @dataclass @@ -478,7 +468,7 @@ class NamedTupleParser(AbstractParser[tuple, NT]): 'field_parsers') hook: Callable[ - [Any, type[tuple], Optional[FieldToParser], List[AbstractParser]], + [Any, type[tuple], Optional['FieldToParser'], List[AbstractParser]], NT ] get_parser: InitVar[GetParserType] @@ -490,7 +480,7 @@ def __post_init__(self, cls: Type, # Get the field annotations for the `NamedTuple` type type_anns: Dict[str, type[T]] = self.base_type.__annotations__ - self.field_to_parser: Optional[FieldToParser] = { + self.field_to_parser: Optional['FieldToParser'] = { f: get_parser(ftype, cls, extras) for f, ftype in type_anns.items() } @@ -590,14 +580,14 @@ class TypedDictParser(AbstractParser[Type[M], M]): 'optional_keys') base_type: Type[M] - hook: Callable[[Any, Type[M], FieldToParser, FrozenKeys, FrozenKeys], M] + hook: Callable[[Any, Type[M], 'FieldToParser', FrozenKeys, FrozenKeys], M] get_parser: InitVar[GetParserType] def __post_init__(self, cls: Type, extras: Extras, get_parser: GetParserType): - self.key_to_parser: FieldToParser = { + self.key_to_parser: 'FieldToParser' = { k: get_parser(v, cls, extras) for k, v in self.base_type.__annotations__.items() } diff --git a/dataclass_wizard/type_def.py b/dataclass_wizard/type_def.py index db981f9d..d063ac4e 100644 --- a/dataclass_wizard/type_def.py +++ b/dataclass_wizard/type_def.py @@ -16,6 +16,9 @@ 'JSONObject', 'ListOfJSONObject', 'JSONValue', + 'FileType', + 'EnvFileType', + 'StrCollection', 'ParseFloat', 'Encoder', 'FileEncoder', @@ -39,6 +42,7 @@ from collections import deque from datetime import date, time, datetime from enum import Enum +from os import PathLike from typing import ( Any, Type, TypeVar, Sequence, Mapping, List, Dict, DefaultDict, FrozenSet, Union, NamedTuple, Callable, AnyStr, TextIO, BinaryIO, @@ -46,7 +50,7 @@ ForwardRef as PyForwardRef, Literal as PyLiteral, Protocol as PyProtocol, - TypedDict as PyTypedDict, + TypedDict as PyTypedDict, Iterable, Collection, ) from uuid import UUID @@ -119,6 +123,15 @@ # Valid value types in JSON. JSONValue = Union[None, str, bool, int, float, JSONList, JSONObject] +# File-type argument, compatible with the type of `file` for `open` +FileType = Union[str, bytes, PathLike, int] + +# DotEnv file-type argument (string, tuple of string, boolean, or None) +EnvFileType = Union[bool, FileType, Iterable[FileType], None] + +# Type for a string or a collection of strings. +StrCollection = Union[str, Collection[str]] + PyTypedDicts.append(PyTypedDict) # Python 3.9+ users might import from either `typing` or diff --git a/dataclass_wizard/utils/json_util.py b/dataclass_wizard/utils/json_util.py new file mode 100644 index 00000000..80b77b89 --- /dev/null +++ b/dataclass_wizard/utils/json_util.py @@ -0,0 +1,54 @@ +""" +JSON Helper Utilities - *only* internally used in ``errors.py``, +i.e. for rendering exceptions. + +.. NOTE:: + This module should not be imported anywhere at the *top-level* + of another library module! + +""" +__all__ = [ + 'safe_dumps', +] + +from dataclasses import is_dataclass +from datetime import datetime, time, date +from enum import Enum +from json import dumps, JSONEncoder +from typing import Any +from uuid import UUID + +from ..dumpers import asdict + + +class SafeEncoder(JSONEncoder): + """ + A Customized JSON Encoder, which copies core logic in the + `dumpers` module to support serialization of more complex + Python types, such as `datetime` and `Enum`. + """ + + def default(self, o: Any) -> Any: + """Default function, copies the core (minimal) logic from `dumpers.py`.""" + + if is_dataclass(o): + return asdict(o) + + if isinstance(o, Enum): + return o.value + + if isinstance(o, UUID): + return o.hex + + if isinstance(o, (datetime, time)): + return o.isoformat().replace('+00:00', 'Z', 1) + + if isinstance(o, date): + return o.isoformat() + + # anything else (Decimal, timedelta, etc.) + return str(o) + + +def safe_dumps(o, cls=SafeEncoder, **kwargs): + return dumps(o, cls=cls, **kwargs) diff --git a/dataclass_wizard/utils/type_conv.py b/dataclass_wizard/utils/type_conv.py index 7577e1ef..ddcd2b87 100644 --- a/dataclass_wizard/utils/type_conv.py +++ b/dataclass_wizard/utils/type_conv.py @@ -2,6 +2,7 @@ 'as_int', 'as_str', 'as_list', + 'as_dict', 'as_enum', 'as_datetime', 'as_date', @@ -9,9 +10,10 @@ 'as_timedelta', 'date_to_timestamp'] -from datetime import datetime, time, date, timedelta +import json +from datetime import datetime, time, date, timedelta, timezone from numbers import Number -from typing import Union, List, Type, AnyStr, Optional +from typing import Union, Type, AnyStr, Optional, Iterable from ..errors import ParseError from ..lazy_imports import pytimeparse @@ -105,20 +107,36 @@ def as_str(o: Union[str, None], base_type=str, raise_=True): return base_type() -def as_list(o: Union[str, List[str]], sep=','): +def as_list(o: Union[str, Iterable], sep=','): """ - Return `o` if already a list. If `o` is None or an empty string, - return an empty list. Otherwise, split the string on `sep` and + Return `o` if already a list. If `o` is a string, split it on `sep` and return the list result. """ - if not o: - return [] + if isinstance(o, str): + if o.lstrip().startswith('['): + return json.loads(o) + else: + return [e.strip() for e in o.split(sep)] - if isinstance(o, list): - return o + return o - return o.split(sep) + +def as_dict(o: Union[str, Iterable], kv_sep='=', sep=','): + """ + Return `o` if already a dict. If `o` is a string, split it on `sep` and + then split each result by `kv_sep`, and return the dict result. + + """ + if isinstance(o, str): + if o.lstrip().startswith('{'): + return json.loads(o) + else: + # noinspection PyTypeChecker + return dict(map(str.strip, pair.split(kv_sep, 1)) + for pair in o.split(sep)) + + return o def as_enum(o: Union[AnyStr, N], @@ -192,7 +210,7 @@ def as_datetime(o: Union[str, Number, datetime], * ``str``: convert datetime strings (in ISO format) via the built-in ``fromisoformat`` method. * ``Number`` (int or float): Convert a numeric timestamp via the - built-in ``fromtimestamp`` method. + built-in ``fromtimestamp`` method, and return a UTC datetime. * ``datetime``: Return object `o` if it's already of this type or sub-type. @@ -214,12 +232,13 @@ def as_datetime(o: Union[str, Number, datetime], if t is str: # Minor performance fix: if it's a string, we don't need to run # the other type checks. - pass + if raise_: + raise # Check `type` explicitly, because `bool` is a sub-class of `int` elif t in NUMBERS: # noinspection PyTypeChecker - return base_type.fromtimestamp(o) + return base_type.fromtimestamp(o, tz=timezone.utc) elif t is base_type: return o @@ -261,7 +280,8 @@ def as_date(o: Union[str, Number, date], if t is str: # Minor performance fix: if it's a string, we don't need to run # the other type checks. - pass + if raise_: + raise # Check `type` explicitly, because `bool` is a sub-class of `int` elif t in NUMBERS: @@ -305,7 +325,8 @@ def as_time(o: Union[str, time], base_type=time, default=None, raise_=True): if t is str: # Minor performance fix: if it's a string, we don't need to run # the other type checks. - pass + if raise_: + raise elif t is base_type: return o diff --git a/dataclass_wizard/utils/typing_compat.py b/dataclass_wizard/utils/typing_compat.py index 8fbcf129..b2d9567f 100644 --- a/dataclass_wizard/utils/typing_compat.py +++ b/dataclass_wizard/utils/typing_compat.py @@ -11,7 +11,7 @@ 'is_generic', 'is_annotated', 'eval_forward_ref', - 'eval_forward_ref_if_needed' + 'eval_forward_ref_if_needed', ] import functools @@ -73,6 +73,8 @@ def is_literal(cls) -> bool: def _process_forward_annotation(base_type): return PyForwardRef(base_type, is_argument=False) + # def _process_forward_annotation(base_type): + # return PyForwardRef(base_type, is_argument=False, is_class=True) def _get_origin(cls, raise_=False): if isinstance(cls, types.UnionType): diff --git a/docs/common_use_cases/wizard_mixins.rst b/docs/common_use_cases/wizard_mixins.rst index b2a71940..501f2516 100644 --- a/docs/common_use_cases/wizard_mixins.rst +++ b/docs/common_use_cases/wizard_mixins.rst @@ -27,6 +27,91 @@ Use Case Use :class:`JSONPyWizard` when you want to prevent the automatic ``camelCase`` conversion of dictionary keys during serialization, keeping them in their original ``snake_case`` format. +:class:`EnvWizard` +~~~~~~~~~~~~~~~~~~ + +The Env Wizard is a standalone Mixin class that can be extended to enable +loading of Environment Variables and ``.env`` files. + +Similar to how ``dataclasses`` work, it supports type hinting and auto-conversion +from strings to annotated field types. However, it does *not* require a +subclass to be instantiated. + +Here is a simple example of usage with Environment Variables: + +.. code:: python3 + + from __future__ import annotations # can be removed in Python 3.9+ + + from os import environ + from datetime import datetime, time + from typing import NamedTuple + try: + from typing import TypedDict + except ImportError: + from typing_extensions import TypedDict + + from dataclass_wizard import EnvWizard + + # ideally these variables will be set in the environment, like so: + # $ export MY_FLOAT=1.23 + + environ.update( + myStr='Hello', + my_float='432.1', + # lists/dicts can also be specified in JSON format + MyTuple='[1, "2"]', + Keys='{ "k1": "false", "k2": "true" }', + # or in shorthand format... + MY_PENCIL='sharpened=Y, uses_left = 3', + My_Emails=' first_user@abc.com , second-user@xyz.org', + SOME_DT_VAL='1651077045', # 2022-04-27T12:30:45 + ) + + + class Pair(NamedTuple): + first: str + second: int + + + class Pencil(TypedDict): + sharpened: bool + uses_left: int + + + class MyClass(EnvWizard): + + class _(EnvWizard.Meta): + field_to_env_var = { + 'my_dt': 'SOME_DT_VAL', + } + + my_str: str + my_float: float + my_tuple: Pair + keys: dict[str, bool] + my_pencil: Pencil + my_emails: list[str] + my_dt: datetime + my_time: time = time.min + + + print('Class Fields:') + print(MyClass.dict()) + # {'my_str': 'Hello', 'my_float': 432.1, ...} + + print() + + print('JSON:') + print(MyClass.to_json(indent=2)) + # { + # "my_str": "Hello", + # "my_float": 432.1, + # ... + + assert MyClass.my_pencil['uses_left'] == 3 + assert MyClass.my_dt.isoformat() == '2022-04-27T12:30:45' + :class:`JSONListWizard` ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/conf.py b/docs/conf.py index d0dc3293..0663e2ca 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/environ python # # dataclass_wizard documentation build configuration file, created by # sphinx-quickstart on Fri Jun 9 13:47:02 2017. diff --git a/docs/dataclass_wizard.environ.rst b/docs/dataclass_wizard.environ.rst new file mode 100644 index 00000000..4ff0a569 --- /dev/null +++ b/docs/dataclass_wizard.environ.rst @@ -0,0 +1,45 @@ +dataclass\_wizard.environ package +================================= + +Submodules +---------- + +dataclass\_wizard.environ.dumpers module +---------------------------------------- + +.. automodule:: dataclass_wizard.environ.dumpers + :members: + :undoc-members: + :show-inheritance: + +dataclass\_wizard.environ.env\_wizard module +-------------------------------------------- + +.. automodule:: dataclass_wizard.environ.env_wizard + :members: + :undoc-members: + :show-inheritance: + +dataclass\_wizard.environ.loaders module +---------------------------------------- + +.. automodule:: dataclass_wizard.environ.loaders + :members: + :undoc-members: + :show-inheritance: + +dataclass\_wizard.environ.lookups module +---------------------------------------- + +.. automodule:: dataclass_wizard.environ.lookups + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: dataclass_wizard.environ + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/dataclass_wizard.rst b/docs/dataclass_wizard.rst index 4490a4c2..ba638109 100644 --- a/docs/dataclass_wizard.rst +++ b/docs/dataclass_wizard.rst @@ -7,6 +7,7 @@ Subpackages .. toctree:: :maxdepth: 4 + dataclass_wizard.environ dataclass_wizard.utils dataclass_wizard.wizard_cli @@ -85,6 +86,14 @@ dataclass\_wizard.errors module :undoc-members: :show-inheritance: +dataclass\_wizard.helpers module +-------------------------------- + +.. automodule:: dataclass_wizard.helpers + :members: + :undoc-members: + :show-inheritance: + dataclass\_wizard.lazy\_imports module -------------------------------------- diff --git a/docs/dataclass_wizard.utils.rst b/docs/dataclass_wizard.utils.rst index 8d2a7684..cd6c204c 100644 --- a/docs/dataclass_wizard.utils.rst +++ b/docs/dataclass_wizard.utils.rst @@ -20,14 +20,6 @@ dataclass\_wizard.utils.string\_conv module :undoc-members: :show-inheritance: -dataclass\_wizard.utils.type\_check module ------------------------------------------- - -.. automodule:: dataclass_wizard.utils.type_check - :members: - :undoc-members: - :show-inheritance: - dataclass\_wizard.utils.type\_conv module ----------------------------------------- diff --git a/requirements-dev.txt b/requirements-dev.txt index 5a06d3ac..f2e786ca 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,6 +4,7 @@ flake8>=3 # pyup: ignore tox==4.23.2 # Extras pytimeparse==1.1.8 +python-dotenv>=1,<2 # [toml] extra tomli>=2,<3; python_version=="3.9" tomli>=2,<3; python_version=="3.10" diff --git a/setup.py b/setup.py index 13c613e7..e1d40e43 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ """The setup script.""" +import itertools import pathlib from pkg_resources import parse_requirements @@ -32,6 +33,19 @@ test_requirements = [str(req) for req in parse_requirements(requires_test_txt)] else: # Running on CI test_requirements = [] +test_requirements = [ + 'pytest~=7.0.1', + 'pytest-mock~=3.6.1', + 'pytest-cov~=2.12.1', + 'pytest-runner~=5.3.1' +] + +# extras_require = { +# 'dotenv': ['python-dotenv>=0.19.0'], +# } + +# Ref: https://stackoverflow.com/a/71166228/10237506 +# extras_require['all'] = list(itertools.chain.from_iterable(extras_require.values())) about = {} exec((here / package_name / '__version__.py').read_text(), about) @@ -91,6 +105,7 @@ test_suite='tests', tests_require=test_requirements, extras_require={ + 'dotenv': ['python-dotenv>=1,<2'], 'timedelta': ['pytimeparse>=1.1.7'], 'toml': [ 'tomli>=2,<3; python_version=="3.9"', diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 449f5dd5..5b34e6a0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -4,6 +4,8 @@ from dataclasses import dataclass from uuid import UUID +import pytest + @dataclass class SampleClass: @@ -19,3 +21,9 @@ class MyUUIDSubclass(UUID): def __str__(self): return self.hex + + +@pytest.fixture +def mock_log(caplog): + caplog.set_level('INFO', logger='dataclass_wizard') + return caplog diff --git a/tests/unit/environ/.env.prod b/tests/unit/environ/.env.prod new file mode 100644 index 00000000..a6ec35c6 --- /dev/null +++ b/tests/unit/environ/.env.prod @@ -0,0 +1,3 @@ +My_Value=3.21 +# These value overrides the one in another dotenv file (../../.env) +MY_STR='hello world!' diff --git a/tests/unit/environ/.env.test b/tests/unit/environ/.env.test new file mode 100644 index 00000000..6a5544fa --- /dev/null +++ b/tests/unit/environ/.env.test @@ -0,0 +1,3 @@ +myValue=1.23 +Another_Date=1639763585 +my_dt=1651077045 diff --git a/tests/unit/environ/__init__.py b/tests/unit/environ/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/environ/test_dumpers.py b/tests/unit/environ/test_dumpers.py new file mode 100644 index 00000000..6d9c5c26 --- /dev/null +++ b/tests/unit/environ/test_dumpers.py @@ -0,0 +1,19 @@ +import os + +from dataclass_wizard import EnvWizard, json_field + + +def test_dump_with_excluded_fields_and_skip_defaults(): + + os.environ['MY_FIRST_STR'] = 'hello' + os.environ['my-second-str'] = 'world' + + class TestClass(EnvWizard, reload_env=True): + my_first_str: str + my_second_str: str = json_field(..., dump=False) + my_int: int = 123 + + assert TestClass(_reload_env=True).to_dict( + exclude=['my_first_str'], + skip_defaults=True, + ) == {} diff --git a/tests/unit/environ/test_loaders.py b/tests/unit/environ/test_loaders.py new file mode 100644 index 00000000..63f752e2 --- /dev/null +++ b/tests/unit/environ/test_loaders.py @@ -0,0 +1,116 @@ +import os +from collections import namedtuple +from dataclasses import dataclass +from datetime import datetime, date, timezone +from typing import Tuple, NamedTuple, List + +import pytest + +from dataclass_wizard import EnvWizard +from dataclass_wizard.environ.loaders import EnvLoader + + +def test_load_to_bytes(): + assert EnvLoader.load_to_bytes('testing 123', bytes) == b'testing 123' + + +@pytest.mark.parametrize( + 'input,expected', + [ + ('testing 123', bytearray(b'testing 123')), + (b'test', bytearray(b'test')), + ([1, 2, 3], bytearray([1, 2, 3])) + ] +) +def test_load_to_bytearray(input, expected): + assert EnvLoader.load_to_byte_array(input, bytearray) == expected + + +def test_load_to_tuple_and_named_tuple(): + os.environ['MY_TUP'] = '1,2,3' + os.environ['MY_NT'] = '[1.23, "string"]' + os.environ['my_untyped_nt'] = 'hello , world, 123' + + class MyNT(NamedTuple): + my_float: float + my_str: str + + untyped_tup = namedtuple('untyped_tup', ('a', 'b', 'c')) + + class MyClass(EnvWizard, reload_env=True): + my_tup: Tuple[int, ...] + my_nt: MyNT + my_untyped_nt: untyped_tup + + c = MyClass() + + assert c.dict() == {'my_nt': MyNT(my_float=1.23, my_str='string'), + 'my_tup': (1, 2, 3), + 'my_untyped_nt': untyped_tup(a='hello', b='world', c='123')} + + assert c.to_dict() == {'my_nt': MyNT(my_float=1.23, my_str='string'), + 'my_tup': (1, 2, 3), + 'my_untyped_nt': untyped_tup(a='hello', b='world', c='123')} + + +def test_load_to_dataclass(): + """When an `EnvWizard` subclass has a nested dataclass schema.""" + + os.environ['inner_cls_1'] = 'my_bool=false, my_string=test' + os.environ['inner_cls_2'] = '{"answerToLife": "42", "MyList": "testing, 123 , hello!"}' + + @dataclass + class Inner1: + my_bool: bool + my_string: str + + @dataclass + class Inner2: + answer_to_life: int + my_list: List[str] + + class MyClass(EnvWizard, reload_env=True): + + inner_cls_1: Inner1 + inner_cls_2: Inner2 + + c = MyClass() + print(c) + + assert c.dict() == { + 'inner_cls_1': Inner1(my_bool=False, + my_string='test'), + 'inner_cls_2': Inner2(answer_to_life=42, + my_list=['testing', '123', 'hello!']), + } + + assert c.to_dict() == { + 'inner_cls_1': {'my_bool': False, + 'my_string': 'test'}, + 'inner_cls_2': {'answer_to_life': 42, + 'my_list': ['testing', '123', 'hello!']} + } + + +@pytest.mark.parametrize( + 'input,expected', + [ + ('2021-11-28T17:35:55', datetime(2021, 11, 28, 17, 35, 55)), + (1577952245, datetime(2020, 1, 2, 8, 4, 5, tzinfo=timezone.utc)), + (datetime.min, datetime.min) + ] +) +def test_load_to_datetime(input, expected): + assert EnvLoader.load_to_datetime(input, datetime) == expected + + +@pytest.mark.parametrize( + 'input,expected', + [ + ('2021-11-28', date(2021, 11, 28)), + (1577952245, date(2020, 1, 2)), + (date.min, date.min) + ] +) +def test_load_to_date(input, expected): + assert EnvLoader.load_to_date(input, date) == expected diff --git a/tests/unit/environ/test_lookups.py b/tests/unit/environ/test_lookups.py new file mode 100644 index 00000000..ec7db62a --- /dev/null +++ b/tests/unit/environ/test_lookups.py @@ -0,0 +1,72 @@ +import pytest + +from dataclass_wizard.environ.lookups import * + + +@pytest.mark.parametrize( + 'string,expected', + [ + ('device_type', 'devicetype'), + ('isACamelCasedWORD', 'isacamelcasedword'), + ('ATitledWordToTESTWith', 'atitledwordtotestwith'), + ('not-a-tester', 'notatester'), + ('helloworld', 'helloworld'), + ('A', 'a'), + ('TESTing_if_thisWorks', 'testingifthisworks'), + ('a_B_Cde_fG_hi', 'abcdefghi'), + ('How_-Are-_YoUDoing__TeST', 'howareyoudoingtest'), + ] +) +def test_clean(string, expected): + assert clean(string) == expected + + +def test_lookup_exact(): + assert lookup_exact('abc-this-key-shouldnt-exist') is MISSING + assert lookup_exact(('abc-this-key-shouldnt-exist', )) is MISSING + + +def test_reload_when_not_accessed_cleaned_to_env(): + # save current value + current_val = Env._accessed_cleaned_to_env + + Env._accessed_cleaned_to_env = False + Env.reload() + + # don't forget to reset it + Env._accessed_cleaned_to_env = current_val + + +def test_with_snake_case(): + var = 'my_test_string_1' + assert with_snake_case(var) is MISSING + + os.environ['MY_TEST_STRING_1'] = 'hello world' + Env.reload() + assert with_snake_case(var) == 'hello world' + + os.environ[var] = 'testing 123' + Env.reload() + assert with_snake_case(var) == 'testing 123' + + +def test_with_pascal_or_camel_case(): + var = 'MyTestString2' + assert with_pascal_or_camel_case(var) is MISSING + + os.environ['my_test_string2'] = 'testing 123' + Env.reload() + assert with_pascal_or_camel_case(var) == 'testing 123' + + os.environ['MY_TEST_STRING2'] = 'hello world' + Env.reload() + assert with_pascal_or_camel_case(var) == 'hello world' + + if os.name == 'nt': + # Windows: var names are automatically converted + # to upper case when saved to `os.environ` + return + + os.environ[var] = 'hello world !!' + Env.reload() + assert with_pascal_or_camel_case(var) == 'hello world !!' diff --git a/tests/unit/environ/test_wizard.py b/tests/unit/environ/test_wizard.py new file mode 100644 index 00000000..9af1c2b2 --- /dev/null +++ b/tests/unit/environ/test_wizard.py @@ -0,0 +1,439 @@ +import logging +import os +from dataclasses import field +from datetime import datetime, time, date, timezone +from pathlib import Path +from textwrap import dedent +from typing import ClassVar, List, Dict, Union, DefaultDict, Set + +import pytest + +from dataclass_wizard import EnvWizard, json_field +from dataclass_wizard.errors import MissingVars, ParseError, ExtraData + +from ...conftest import * + + +log = logging.getLogger(__name__) + +# quick access to the `tests/unit` directory +here = Path(__file__).parent + + +def test_load_and_dump(): + """Basic example with simple types (str, int) and collection types such as list.""" + + os.environ.update({ + 'hello_world': 'Test', + 'MyStr': 'This STRING', + 'MY_TEST_VALUE123': '11', + 'THIS_Num': '23', + 'my_list': '["1", 2, "3", "4.5", 5.7]', + 'my_other_list': 'rob@test.org, this@email.com , hello-world_123@tst.org,z@ab.c' + }) + + class MyClass(EnvWizard, reload_env=True): + # these are class-level fields, and should be ignored + my_cls_var: ClassVar[str] + other_var = 21 + + my_str: str + this_num: int + my_list: List[int] + my_other_list: List[str] + my_test_value123: int = 21 + # missing from environment + my_field_not_in_env: str = 'testing' + + e = MyClass() + log.debug(e.dict()) + + assert not hasattr(e, 'my_cls_var') + assert e.other_var == 21 + + assert e.my_str == 'This STRING' + assert e.this_num == 23 + assert e.my_list == [1, 2, 3, 4, 6] + assert e.my_other_list == ['rob@test.org', 'this@email.com', 'hello-world_123@tst.org', 'z@ab.c'] + assert e.my_test_value123 == 11 + assert e.my_field_not_in_env == 'testing' + + assert e.to_dict() == { + 'my_str': 'This STRING', + 'this_num': 23, + 'my_list': [1, 2, 3, 4, 6], + 'my_other_list': ['rob@test.org', + 'this@email.com', + 'hello-world_123@tst.org', + 'z@ab.c'], + 'my_test_value123': 11, + 'my_field_not_in_env': 'testing', + } + + +def test_load_and_dump_with_dict(): + """Example with more complex types such as dict, TypedDict, and defaultdict.""" + + os.environ.update({ + 'MY_DICT': '{"123": "True", "5": "false"}', + 'My.Other.Dict': 'some_key=value, anotherKey=123 ,LastKey=just a test~', + 'My_Default_Dict': ' { "1.2": "2021-01-02T13:57:21" } ', + 'myTypedDict': 'my_bool=true' + }) + + class MyTypedDict(TypedDict): + my_bool: bool + + # Fix so the forward reference works + globals().update(locals()) + + class ClassWithDict(EnvWizard, reload_env=True): + class _(EnvWizard.Meta): + field_to_env_var = {'my_other_dict': 'My.Other.Dict'} + + my_dict: Dict[int, bool] + my_other_dict: Dict[str, Union[int, str]] + my_default_dict: DefaultDict[float, datetime] + my_typed_dict: MyTypedDict + + c = ClassWithDict() + log.debug(c.dict()) + + assert c.my_dict == {123: True, 5: False} + + # note that the value for 'anotherKey' is a string value ('123') here, + # but we might want to see if we can update it to a numeric value (123) + # instead. + assert c.my_other_dict == { + 'some_key': 'value', + 'anotherKey': '123', + 'LastKey': 'just a test~', + } + + assert c.my_default_dict == {1.2: datetime(2021, 1, 2, 13, 57, 21)} + assert c.my_typed_dict == {'my_bool': True} + + assert c.to_dict() == { + 'my_dict': {5: False, 123: True}, + 'my_other_dict': {'LastKey': 'just a test~', + 'anotherKey': '123', + 'some_key': 'value'}, + 'my_default_dict': {1.2: '2021-01-02T13:57:21'}, + 'my_typed_dict': {'my_bool': True} + } + + +def test_load_and_dump_with_aliases(): + """ + Example with fields that are aliased to differently-named env variables + in the Environment. + """ + + os.environ.update({ + 'hello_world': 'Test', + 'MY_TEST_VALUE123': '11', + 'the_number': '42', + 'my_list': '3, 2, 1,0', + 'My_Other_List': 'rob@test.org, this@email.com , hello-world_123@tst.org,z@ab.c' + }) + + class MyClass(EnvWizard, reload_env=True): + class _(EnvWizard.Meta): + field_to_env_var = { + 'answer_to_life': 'the_number', + 'emails': ('EMAILS', 'My_Other_List'), + } + + my_str: str = json_field(('the_string', 'hello_world')) + answer_to_life: int + list_of_nums: List[int] = json_field('my_list') + emails: List[str] + # added for code coverage. + # case where `json_field` is used, but an alas is not defined. + my_test_value123: int = json_field(..., default=21) + + c = MyClass() + log.debug(c.dict()) + + assert c.my_str == 'Test' + assert c.answer_to_life == 42 + assert c.list_of_nums == [3, 2, 1, 0] + assert c.emails == ['rob@test.org', 'this@email.com', 'hello-world_123@tst.org', 'z@ab.c'] + assert c.my_test_value123 == 11 + + assert c.to_dict() == { + 'answer_to_life': 42, + 'emails': ['rob@test.org', + 'this@email.com', + 'hello-world_123@tst.org', + 'z@ab.c'], + 'list_of_nums': [3, 2, 1, 0], + 'my_str': 'Test', + 'my_test_value123': 11, + } + + +def test_load_with_missing_env_variables(): + """ + Test calling the constructor of an `EnvWizard` subclass when the + associated vars are missing in the Environment. + """ + + class MyClass(EnvWizard): + missing_field_1: str + missing_field_2: datetime + missing_field_3: Dict[str, int] + default_field: Set[str] = field(default_factory=set) + + with pytest.raises(MissingVars) as e: + _ = MyClass() + + assert str(e.value) == dedent("""\ + There are 3 required fields in class `test_load_with_missing_env_variables..MyClass` missing in the Environment: + - missing_field_1 + - missing_field_2 + - missing_field_3 + + resolution #1: set a default value for any optional fields, as below. + + class test_load_with_missing_env_variables..MyClass: + missing_field_1: str = '' + missing_field_2: datetime = None + missing_field_3: typing.Dict[str, int] = None + + ... + resolution #2: pass in values for required fields to test_load_with_missing_env_variables..MyClass.__init__(): + + instance = test_load_with_missing_env_variables..MyClass(missing_field_1='', missing_field_2=None, missing_field_3=None) + """.rstrip()) + + # added for code coverage. + # test when only missing a single (1) required field. + with pytest.raises(MissingVars) as e: + _ = MyClass(missing_field_1='test', missing_field_3='key=123') + + error_info = str(e.value) + assert '1 required field' in error_info + assert 'missing_field_2' in error_info + + +def test_load_with_parse_error(): + os.environ.update(MY_STR='abc') + + class MyClass(EnvWizard, reload_env=True): + class _(EnvWizard.Meta): + debug_enabled = True + + my_str: int + + with pytest.raises(ParseError) as e: + _ = MyClass() + + assert str(e.value.base_error) == "invalid literal for int() with base 10: 'abc'" + assert e.value.kwargs['env_variable'] == 'MY_STR' + + +def test_load_with_parse_error_when_env_var_is_specified(): + """ + Raising `ParseError` when a dataclass field to env var mapping is + specified. Added for code coverage. + """ + + os.environ.update(MY_STR='abc') + + class MyClass(EnvWizard, reload_env=True): + class _(EnvWizard.Meta): + debug_enabled = True + + a_string: int = json_field('MY_STR') + + with pytest.raises(ParseError) as e: + _ = MyClass() + + assert str(e.value.base_error) == "invalid literal for int() with base 10: 'abc'" + assert e.value.kwargs['env_variable'] == 'MY_STR' + + +def test_load_with_dotenv_file(): + """Test reading from the `.env` file in project root directory.""" + + class MyClass(EnvWizard): + class _(EnvWizard.Meta): + env_file = True + + my_str: int + my_time: time + my_date: date = None + + assert MyClass().dict() == {'my_str': 42, + 'my_time': time(15, 20), + 'my_date': date(2022, 1, 21)} + + +def test_load_with_dotenv_file_with_path(): + """Test reading from the `.env.test` file in `tests/unit` directory.""" + + class MyClass(EnvWizard): + class _(EnvWizard.Meta): + env_file = here / '.env.test' + key_lookup_with_load = 'PASCAL' + + my_value: float + my_dt: datetime + another_date: date + + c = MyClass() + + assert c.dict() == {'my_value': 1.23, + 'my_dt': datetime(2022, 4, 27, 16, 30, 45, tzinfo=timezone.utc), + 'another_date': date(2021, 12, 17)} + + expected_json = '{"another_date": "2021-12-17", "my_dt": "2022-04-27T16:30:45Z", "my_value": 1.23}' + assert c.to_json(sort_keys=True) == expected_json + + +def test_load_with_tuple_of_dotenv_and_env_file_param_to_init(): + """ + Test when `env_file` is specified as a tuple of dotenv files, and + the `_env_file` parameter is also passed in to the constructor + or __init__() method. + """ + + os.environ.update( + MY_STR='default from env', + myValue='3322.11', + Other_Key='5', + ) + + class MyClass(EnvWizard): + class _(EnvWizard.Meta): + env_file = '.env', here / '.env.test' + key_lookup_with_load = 'PASCAL' + + my_value: float + my_str: str + other_key: int = 3 + + # pass `_env_file=False` so we don't load the Meta `env_file` + c = MyClass(_env_file=False, _reload_env=True) + + assert c.dict() == {'my_str': 'default from env', + 'my_value': 3322.11, + 'other_key': 5} + + # load variables from the Meta `env_file` tuple, and also pass + # in `other_key` to the constructor method. + c = MyClass(other_key=7) + + assert c.dict() == {'my_str': '42', + 'my_value': 1.23, + 'other_key': 7} + + # load variables from the `_env_file` argument to the constructor + # method, overriding values from `env_file` in the Meta config. + c = MyClass(_env_file=here / '.env.prod') + + assert c.dict() == {'my_str': 'hello world!', + 'my_value': 3.21, + 'other_key': 5} + + +def test_load_when_constructor_kwargs_are_passed(): + """ + Using the constructor method of an `EnvWizard` subclass when + passing keyword arguments instead of the Environment. + """ + os.environ.update(MY_STRING_VAR='hello world') + + class MyTestClass(EnvWizard, reload_env=True): + my_string_var: str + + c = MyTestClass(my_string_var='test!!') + assert c.my_string_var == 'test!!' + + c = MyTestClass() + assert c.my_string_var == 'hello world' + + +def test_extra_keyword_arguments_when_deny_extra(): + """ + Passing extra keyword arguments to the constructor method of an `EnvWizard` + subclass raises an error by default, as `Extra.DENY` is the default behavior. + """ + + os.environ['A_FIELD'] = 'hello world!' + + class MyClass(EnvWizard, reload_env=True): + a_field: str + + with pytest.raises(ExtraData) as e: + _ = MyClass(another_field=123, third_field=None) + + log.error(e.value) + + +def test_extra_keyword_arguments_when_allow_extra(): + """ + Passing extra keyword arguments to the constructor method of an `EnvWizard` + subclass does not raise an error and instead accepts or "passes through" + extra keyword arguments, when `Extra.ALLOW` is specified for the + `extra` Meta field. + """ + + os.environ['A_FIELD'] = 'hello world!' + + class MyClass(EnvWizard, reload_env=True): + + class _(EnvWizard.Meta): + extra = 'ALLOW' + + a_field: str + + c = MyClass(another_field=123, third_field=None) + + assert getattr(c, 'another_field') == 123 + assert hasattr(c, 'third_field') + + assert c.to_json() == '{"a_field": "hello world!"}' + + +def test_extra_keyword_arguments_when_ignore_extra(): + """ + Passing extra keyword arguments to the constructor method of an `EnvWizard` + subclass does not raise an error and instead ignores extra keyword + arguments, when `Extra.IGNORE` is specified for the `extra` Meta field. + """ + + os.environ['A_FIELD'] = 'hello world!' + + class MyClass(EnvWizard, reload_env=True): + + class _(EnvWizard.Meta): + extra = 'IGNORE' + + a_field: str + + c = MyClass(another_field=123, third_field=None) + + assert not hasattr(c, 'another_field') + assert not hasattr(c, 'third_field') + + assert c.to_json() == '{"a_field": "hello world!"}' + + +def test_init_method_declaration_is_logged_when_debug_mode_is_enabled(mock_log): + + class _EnvSettings(EnvWizard): + + class _(EnvWizard.Meta): + debug_enabled = True + extra = 'ALLOW' + + auth_key: str = json_field('my_auth_key') + api_key: str = json_field(('hello', 'test')) + domains: Set[str] = field(default_factory=set) + answer_to_life: int = 42 + + # assert that the __init__() method declaration is logged + assert mock_log.records[-1].levelname == 'INFO' + assert '_EnvSettings.__init__()' in mock_log.records[-1].message diff --git a/tests/unit/test_bases_meta.py b/tests/unit/test_bases_meta.py index 85909be6..87a92562 100644 --- a/tests/unit/test_bases_meta.py +++ b/tests/unit/test_bases_meta.py @@ -1,14 +1,14 @@ import logging from dataclasses import dataclass, field from datetime import datetime, date -from typing import Optional, List, Type +from typing import Optional, List from unittest.mock import ANY import pytest from pytest_mock import MockerFixture -from dataclass_wizard import JSONWizard from dataclass_wizard.bases import META +from dataclass_wizard import JSONWizard, EnvWizard from dataclass_wizard.bases_meta import BaseJSONWizardMeta from dataclass_wizard.enums import LetterCase, DateTimeTo from dataclass_wizard.errors import ParseError @@ -18,11 +18,6 @@ log = logging.getLogger(__name__) -@pytest.fixture -def mock_log(mocker: MockerFixture): - return mocker.patch('dataclass_wizard.bases_meta.LOG') - - @pytest.fixture def mock_meta_initializers(mocker: MockerFixture): return mocker.patch('dataclass_wizard.bases_meta.META_INITIALIZER') @@ -34,6 +29,12 @@ def mock_bind_to(mocker: MockerFixture): 'dataclass_wizard.bases_meta.BaseJSONWizardMeta.bind_to') +@pytest.fixture +def mock_env_bind_to(mocker: MockerFixture): + return mocker.patch( + 'dataclass_wizard.bases_meta.BaseEnvWizardMeta.bind_to') + + @pytest.fixture def mock_get_dumper(mocker: MockerFixture): return mocker.patch('dataclass_wizard.bases_meta.get_dumper') @@ -150,7 +151,7 @@ class Meta(JSONWizard.Meta): isActive: bool = False myDt: Optional[datetime] = None - mock_log.info.assert_called_once_with('DEBUG Mode is enabled') + assert 'DEBUG Mode is enabled' in mock_log.text string = """ { @@ -191,7 +192,7 @@ class Meta(JSONWizard.Meta): assert d['my_dt'] == round(expected_dt.timestamp()) -def test_json_key_to_field_when_add_is_a_falsy_value(mock_log): +def test_json_key_to_field_when_add_is_a_falsy_value(): """ The `json_key_to_field` attribute is specified when subclassing :class:`JSONWizard.Meta`, but the `__all__` field a falsy value. @@ -213,7 +214,7 @@ class Meta(JSONWizard.Meta): myCustomStr: str # note: this is only expected to run at most once - # mock_log.info.assert_called_once_with('DEBUG Mode is enabled') + # assert 'DEBUG Mode is enabled' in mock_log.text string = """ { @@ -236,7 +237,7 @@ class Meta(JSONWizard.Meta): assert d['my_custom_str'] == "test that this is mapped to 'myCustomStr'" -def test_meta_config_is_not_implicitly_shared_between_dataclasses(mock_log): +def test_meta_config_is_not_implicitly_shared_between_dataclasses(): @dataclass class MyFirstClass(JSONWizard): @@ -323,6 +324,20 @@ class _(JSONWizard.Meta): mock_meta_initializers.__setitem__.assert_called_once() +def test_env_meta_initializer_not_called_when_meta_is_not_an_inner_class( + mock_meta_initializers, mock_env_bind_to): + """ + Meta Initializer `dict` should *not* be updated when `Meta` has no outer + class. + """ + + class _(EnvWizard.Meta): + debug_enabled = True + + mock_meta_initializers.__setitem__.assert_not_called() + mock_env_bind_to.assert_called_once_with(ANY, create=False) + + def test_meta_initializer_not_called_when_meta_is_not_an_inner_class( mock_meta_initializers, mock_bind_to): """ @@ -337,8 +352,7 @@ class _(JSONWizard.Meta): mock_bind_to.assert_called_once_with(ANY, create=False) -def test_meta_initializer_errors_when_key_transform_with_load_is_invalid( - mock_log): +def test_meta_initializer_errors_when_key_transform_with_load_is_invalid(): """ Test when an invalid value for the ``key_transform_with_load`` attribute is specified when sub-classing from :class:`JSONWizard.Meta`. @@ -355,8 +369,7 @@ class Meta(JSONWizard.Meta): list_of_int: List[int] = field(default_factory=list) -def test_meta_initializer_errors_when_key_transform_with_dump_is_invalid( - mock_log): +def test_meta_initializer_errors_when_key_transform_with_dump_is_invalid(): """ Test when an invalid value for the ``key_transform_with_dump`` attribute is specified when sub-classing from :class:`JSONWizard.Meta`. @@ -373,8 +386,7 @@ class Meta(JSONWizard.Meta): list_of_int: List[int] = field(default_factory=list) -def test_meta_initializer_errors_when_marshal_date_time_as_is_invalid( - mock_log): +def test_meta_initializer_errors_when_marshal_date_time_as_is_invalid(): """ Test when an invalid value for the ``marshal_date_time_as`` attribute is specified when sub-classing from :class:`JSONWizard.Meta`. @@ -391,8 +403,7 @@ class Meta(JSONWizard.Meta): list_of_int: List[int] = field(default_factory=list) -def test_meta_initializer_is_noop_when_marshal_date_time_as_is_iso_format( - mock_log, mock_get_dumper): +def test_meta_initializer_is_noop_when_marshal_date_time_as_is_iso_format(mock_get_dumper): """ Test that it's a noop when the value for ``marshal_date_time_as`` is `ISO_FORMAT`, which is the default conversion method for the dumper diff --git a/tests/unit/test_load.py b/tests/unit/test_load.py index 6ab44d53..54335aec 100644 --- a/tests/unit/test_load.py +++ b/tests/unit/test_load.py @@ -25,6 +25,7 @@ OptionalParser, Parser, IdentityParser, SingleArgParser ) from dataclass_wizard.type_def import NoneType, T + from .conftest import MyUUIDSubclass from ..conftest import * @@ -574,7 +575,7 @@ class MyClass(JSONSerializable): 'input,expected,expectation', [ ([1, '2', 3], {1, 2, 3}, does_not_raise()), - ('TrUe', True, pytest.raises(ParseError)), + ('TrUe', True, pytest.raises(ValueError)), ((3.22, 2.11, 1.22), {3, 2, 1}, does_not_raise()), ] ) @@ -602,7 +603,7 @@ class MyClass(JSONSerializable): 'input,expected,expectation', [ ([1, '2', 3], {1, 2, 3}, does_not_raise()), - ('TrUe', True, pytest.raises(ParseError)), + ('TrUe', True, pytest.raises(ValueError)), ((3.22, 2.11, 1.22), {1, 2, 3}, does_not_raise()), ] ) @@ -798,7 +799,7 @@ class C: @pytest.mark.parametrize( 'input,expectation', [ - ('testing', pytest.raises(TypeError)), + ('testing', pytest.raises(ValueError)), ('2020-01-02T01:02:03Z', does_not_raise()), ('2010-12-31 23:59:59-04:00', does_not_raise()), (123456789, does_not_raise()), @@ -822,7 +823,7 @@ class MyClass(JSONSerializable): @pytest.mark.parametrize( 'input,expectation', [ - ('testing', pytest.raises(TypeError)), + ('testing', pytest.raises(ValueError)), ('2020-01-02', does_not_raise()), ('2010-12-31', does_not_raise()), (123456789, does_not_raise()), @@ -846,7 +847,7 @@ class MyClass(JSONSerializable): @pytest.mark.parametrize( 'input,expectation', [ - ('testing', pytest.raises(TypeError)), + ('testing', pytest.raises(ValueError)), ('01:02:03Z', does_not_raise()), ('23:59:59-04:00', does_not_raise()), (123456789, pytest.raises(TypeError)), @@ -926,7 +927,7 @@ class _(JSONSerializable.Meta): [1, '2', 3], does_not_raise(), [1, 2, 3] ), ( - 'testing', pytest.raises(ParseError), None + 'testing', pytest.raises(ValueError), None ), ] ) @@ -949,7 +950,7 @@ class MyClass(JSONSerializable): 'input,expectation,expected', [ ( - ['hello', 'world'], pytest.raises(ParseError), None + ['hello', 'world'], pytest.raises(ValueError), None ), ( [1, '2', 3], does_not_raise(), [1, 2, 3] @@ -1238,7 +1239,7 @@ class MyClass(JSONSerializable): # Might need to change this behavior if needed: currently it # raises an error, which I think is good for now since we don't # want to add `null`s to a list anyway. - {2: None}, pytest.raises(ParseError), None + {2: None}, pytest.raises(TypeError), None ), ( # Incorrect type - `list`, but should be a `dict` diff --git a/tox.ini b/tox.ini index a77bdf9b..8ac185ba 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,7 @@ deps = ; -r{toxinidir}/requirements.txt commands = pip install -U pip + pip install -e .[all] pytest --basetemp={envtmpdir} # commands = pytest -s --cov-report=term-missing tests