From bd83cc1c3530034bd9ceac35534a6140609a1c06 Mon Sep 17 00:00:00 2001 From: Robin Klaassen Date: Mon, 7 Sep 2020 20:32:36 +0200 Subject: [PATCH 01/10] implemented exclude_predicate --- barentsz/_discover.py | 10 +++++++--- barentsz/_typings.py | 3 +++ tests/test_discover_classes.py | 18 +++++++++++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 barentsz/_typings.py diff --git a/barentsz/_discover.py b/barentsz/_discover.py index 85ccc31..eedb525 100644 --- a/barentsz/_discover.py +++ b/barentsz/_discover.py @@ -22,6 +22,7 @@ from barentsz._here import here from barentsz._attribute import Attribute +from barentsz._typings import exclude_pred def discover( @@ -161,7 +162,7 @@ def discover_classes( include_privates: bool = False, in_private_modules: bool = False, raise_on_fail: bool = False, - exclude: Union[Iterable[type], type] = None + exclude: Union[type, exclude_pred, Iterable[Union[type, exclude_pred]]] = None ) -> List[type]: """ Discover any classes within the given source and according to the given @@ -174,8 +175,8 @@ def discover_classes( in_private_modules: if True, private modules are explored as well. raise_on_fail: if True, raises an ImportError upon the first import failure. - exclude: a type or multiple types that are to be excluded from the - result. + exclude: one or more types or predicates that are to be excluded + from the result. Returns: a list of all discovered classes (types). @@ -186,6 +187,9 @@ def discover_classes( result = list({cls for cls in elements if (signature is Any or subclass_of(cls, signature)) and cls not in exclude_}) + exclude_predicates = [e for e in exclude_ if inspect.isfunction(e)] + for pred in exclude_predicates: + result = [cls for cls in result if not pred(cls)] result.sort(key=lambda cls: cls.__name__) return result diff --git a/barentsz/_typings.py b/barentsz/_typings.py new file mode 100644 index 0000000..454d94b --- /dev/null +++ b/barentsz/_typings.py @@ -0,0 +1,3 @@ +from typing import Callable + +exclude_pred = Callable[[type], bool] diff --git a/tests/test_discover_classes.py b/tests/test_discover_classes.py index c2fc32a..86bf9bc 100644 --- a/tests/test_discover_classes.py +++ b/tests/test_discover_classes.py @@ -80,7 +80,7 @@ def test_discover_classes_with_wrong_argument(self): with self.assertRaises(ValueError): discover_classes(123) - def test_discover_classes_with_exclusions(self): + def test_discover_classes_with_type_exclusions(self): # SETUP path_to_resources = (Path(__file__).parent.parent / 'test_resources' / 'examples_for_tests') @@ -94,3 +94,19 @@ def test_discover_classes_with_exclusions(self): self.assertIn(Class1_level2, classes1) self.assertEqual(1, len(classes2)) self.assertIn(Class1_level2, classes2) + + def test_discover_classes_with_predicate_exclusions(self): + # SETUP + path_to_resources = (Path(__file__).parent.parent / 'test_resources' + / 'examples_for_tests') + + def _name_is_class1(cls: type) -> bool: + return cls.__name__.lower() == 'class1' + + # EXECUTE + classes1 = discover_classes(path_to_resources, exclude=_name_is_class1) + classes2 = discover_classes(path_to_resources, exclude=[_name_is_class1]) + + # VERIFY + self.assertEqual(0, len(classes1)) + self.assertEqual(0, len(classes2)) From 1f23b8d380503f6389f3fed624912912988bffe4 Mon Sep 17 00:00:00 2001 From: Ramon Date: Sun, 20 Sep 2020 21:13:00 +0200 Subject: [PATCH 02/10] Preparations for release 1.2.0 --- .github/workflows/pythonapp.yml | 41 +++++++++++--------- README.md | 4 +- SConstruct.py | 68 +++++++++++++++++++++++++++++++++ barentsz/__init__.py | 18 ++++----- barentsz/_discover.py | 18 +++++---- barentsz/_meta.py | 2 +- setup.cfg | 11 +++++- setup.py | 9 ++++- 8 files changed, 130 insertions(+), 41 deletions(-) create mode 100644 SConstruct.py diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 46b5659..261b16a 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -24,17 +24,32 @@ jobs: pip install .[test] - name: Test - run: | - python setup.py test + run: scons test - name: Doctest if: ${{ matrix.python-version == '3.8' && matrix.os == 'ubuntu-latest' }} - run: | - python -m doctest README.md - + run: scons doctest + + - name: Pycodestyle + run: scons pycodestyle + + - name: Pylint + run: scons pylint + + - name: Imports + run: scons check format + + - name: Mypy + run: scons mypy + + - name: Code Complexity + run: scons complexity + + - name: Coverage + run: scons coverage + - name: Generate coverage report - run: | - pytest --cov=./ --cov-report=xml + run: pytest --cov=./ --cov-report=xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 @@ -46,15 +61,3 @@ jobs: env_vars: OS,PYTHON name: codecov-umbrella fail_ci_if_error: true - - - name: Pycodestyle - run: | - pycodestyle barentsz -v - - - name: Pylint - run: | - pylint --rcfile=setup.cfg barentsz - - - name: Mypy - run: | - mypy barentsz --disallow-untyped-defs diff --git a/README.md b/README.md index 5716c2a..303a9cb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ [![Python versions](https://img.shields.io/pypi/pyversions/barentsz.svg)](https://img.shields.io/pypi/pyversions/barentsz.svg) [![PyPI version](https://badge.fury.io/py/barentsz.svg)](https://badge.fury.io/py/barentsz) -![barentsz](https://github.com/ramonhagenaars/barentsz/workflows/barentsz/badge.svg) [![codecov](https://codecov.io/gh/ramonhagenaars/barentsz/branch/master/graph/badge.svg)](https://codecov.io/gh/ramonhagenaars/barentsz) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/ramonhagenaars/barentsz/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/ramonhagenaars/barentsz/?branch=master) @@ -366,6 +365,9 @@ discover_paths(directory: Union[pathlib.Path, str], pattern: str) -> List[pathli ## ❄ Changelist +### 1.2.0 [2020-09-20] +* Changed `exclude` parameter to also allow predicates. + ### 1.1.0 [2020-08-05] * Added the `here` function that returns the directory of the caller of that function. * Added the `discovery` function that can conveniently find types using the current dir and a given class. diff --git a/SConstruct.py b/SConstruct.py new file mode 100644 index 0000000..d939c98 --- /dev/null +++ b/SConstruct.py @@ -0,0 +1,68 @@ +import subprocess + +_CHECK_ONLY = 'check' in COMMAND_LINE_TARGETS +_CONTINUE = 'continue' in COMMAND_LINE_TARGETS +_SUBJECT = 'barentsz' + + +def _exec(cmd: str) -> None: + print(f'>>> {cmd}') + exit_code = subprocess.call(cmd, shell=True) + if exit_code != 0 and not _CONTINUE: + print(f'Exiting with {exit_code}') + Exit(exit_code) + + +# COMBINATIONS: + +if 'all' in COMMAND_LINE_TARGETS: + COMMAND_LINE_TARGETS += ['format', 'quality'] + +if 'quality' in COMMAND_LINE_TARGETS: + COMMAND_LINE_TARGETS += ['test', 'doctest', 'coverage', 'pycodestyle', 'pylint', 'mypy', 'complexity'] + +if 'format' in COMMAND_LINE_TARGETS: + COMMAND_LINE_TARGETS += ['autoflake', 'isort'] + + +# QUALITY: + +if 'test' in COMMAND_LINE_TARGETS: + _exec('python -m unittest discover tests') + +if 'doctest' in COMMAND_LINE_TARGETS: + _exec('python -m doctest README.md') + +if 'coverage' in COMMAND_LINE_TARGETS: + _exec('coverage run -m unittest discover tests') + _exec('coverage report -m --fail-under=100') + +if 'pycodestyle' in COMMAND_LINE_TARGETS: + _exec(f'pycodestyle {_SUBJECT} -v --config=setup.cfg') + +if 'pylint' in COMMAND_LINE_TARGETS: + _exec(f'pylint --rcfile=setup.cfg {_SUBJECT}') + +if 'mypy' in COMMAND_LINE_TARGETS: + _exec(f'mypy {_SUBJECT} --show-error-codes --disallow-untyped-defs') + +if 'complexity' in COMMAND_LINE_TARGETS: + _exec(f'radon cc {_SUBJECT} -nc --total-average') + _exec(f'xenon {_SUBJECT} --max-absolute B --max-modules A --max-average A') + + +# FORMAT: + +if 'autoflake' in COMMAND_LINE_TARGETS: + cmd = f'autoflake {_SUBJECT}/_here.py --recursive --in-place --remove-unused-variables --expand-star-imports' + if _CHECK_ONLY: + cmd += ' --check' + _exec(cmd) + +if 'isort' in COMMAND_LINE_TARGETS: + cmd = f'isort {_SUBJECT} --recursive --quiet' + if _CHECK_ONLY: + cmd += ' --check' + _exec(cmd) + +Exit(0) diff --git a/barentsz/__init__.py b/barentsz/__init__.py index 698522a..b96f6f7 100644 --- a/barentsz/__init__.py +++ b/barentsz/__init__.py @@ -1,16 +1,12 @@ from barentsz._discover import ( discover, - discover_paths, - discover_packages, - discover_module_names, - discover_modules, + discover_attributes, discover_classes, discover_functions, - discover_attributes, -) -from barentsz._here import ( - here, -) -from barentsz._meta import ( - __version__, + discover_module_names, + discover_modules, + discover_packages, + discover_paths, ) +from barentsz._here import here +from barentsz._meta import __version__ diff --git a/barentsz/_discover.py b/barentsz/_discover.py index 85ccc31..3f03765 100644 --- a/barentsz/_discover.py +++ b/barentsz/_discover.py @@ -5,23 +5,27 @@ from importlib import import_module from pathlib import Path from typing import ( - Union, - Dict, - List, Any, Callable, - Type, + Dict, Iterable, + List, Optional, - Tuple, Set, + Tuple, + Type, TypeVar, + Union, ) -from typish import Module, subclass_of, instance_of +from typish import ( + Module, + instance_of, + subclass_of, +) -from barentsz._here import here from barentsz._attribute import Attribute +from barentsz._here import here def discover( diff --git a/barentsz/_meta.py b/barentsz/_meta.py index efe6c44..683f8d2 100644 --- a/barentsz/_meta.py +++ b/barentsz/_meta.py @@ -1,5 +1,5 @@ __title__ = 'barentsz' -__version__ = '1.1.0' +__version__ = '1.2.0' __author__ = 'Ramon Hagenaars' __author_email__ = 'ramon.hagenaars@gmail.com' __description__ = 'For discovering modules, classes, functions and attributes.' diff --git a/setup.cfg b/setup.cfg index cb2e331..922608f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,4 +9,13 @@ disable = too-many-arguments, missing-module-docstring, no-value-for-parameter, - isinstance-second-argument-not-valid-type + isinstance-second-argument-not-valid-type, + +[coverage:run] +include = barentsz/* + +[isort] +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 3 +use_parentheses = True diff --git a/setup.py b/setup.py index cfcfe3a..0b0524e 100644 --- a/setup.py +++ b/setup.py @@ -14,14 +14,21 @@ requirements = [ 'typish>=1.7.0', -], +] test_requirements = [ + 'scons', 'pycodestyle', 'pylint', 'mypy', 'pytest', 'pytest-cov', + 'coverage', + 'radon', + 'xenon', + 'autoflake', + 'isort', + ] extras = { From ee95758006929b3ec540ffcecc1046e888c64125 Mon Sep 17 00:00:00 2001 From: Ramon Date: Sun, 20 Sep 2020 21:21:11 +0200 Subject: [PATCH 03/10] Updated actions file. --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 261b16a..1429da1 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -1,6 +1,6 @@ name: barentsz -on: [push] +on: [push, pull_request] jobs: build: From 5591825299b4a1c2176f5af28feec89b5e21ad64 Mon Sep 17 00:00:00 2001 From: Ramon Date: Sun, 20 Sep 2020 21:23:34 +0200 Subject: [PATCH 04/10] Added "from" to raise statement. --- barentsz/_discover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barentsz/_discover.py b/barentsz/_discover.py index 3f03765..c3e915e 100644 --- a/barentsz/_discover.py +++ b/barentsz/_discover.py @@ -154,7 +154,7 @@ def discover_modules( result.append(imported_module) except Exception as err: if raise_on_fail: - raise ImportError(err) + raise ImportError(err) from err result.sort(key=lambda module: module.__name__) return result From 9e91be9561d4990c9693cdaf35ff5417f4db450b Mon Sep 17 00:00:00 2001 From: Ramon Date: Sun, 20 Sep 2020 21:31:24 +0200 Subject: [PATCH 05/10] Added ignore file for coverage. --- .coveragerc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index a3685a8..7981e20 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,5 +2,7 @@ [run] omit = setup.py + SConstruct.py + */docs/* + */test_resources/* tests/* - test_resources/* From 357b234e4b3c5e02d71e431ab16e7bed21f31d4e Mon Sep 17 00:00:00 2001 From: Ramon Date: Sun, 20 Sep 2020 21:36:42 +0200 Subject: [PATCH 06/10] Fixed doctest. --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 303a9cb..04784f7 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ discover(source: Any = None, *, what: Any = typing.List[type], **kwargs: dict) - >>> help(discover_classes) Help on function discover_classes in module barentsz._discover: -discover_classes(source: Union[pathlib.Path, str, module, Iterable[module]], signature: type = typing.Any, include_privates: bool = False, in_private_modules: bool = False, raise_on_fail: bool = False, exclude: Union[Iterable[type], type] = None) -> List[type] +discover_classes(source: Union[pathlib.Path, str, module, Iterable[module]], signature: type = typing.Any, include_privates: bool = False, in_private_modules: bool = False, raise_on_fail: bool = False, exclude: Union[type, Callable[[type], bool], Iterable[Union[type, Callable[[type], bool]]]] = None) -> List[type] Discover any classes within the given source and according to the given constraints. @@ -112,13 +112,12 @@ discover_classes(source: Union[pathlib.Path, str, module, Iterable[module]], sig in_private_modules: if True, private modules are explored as well. raise_on_fail: if True, raises an ImportError upon the first import failure. - exclude: a type or multiple types that are to be excluded from the - result. + exclude: one or more types or predicates that are to be excluded + from the result. Returns: a list of all discovered classes (types). - ``` ### Discover Functions From 3a4ac7965eb09e8b98e87bacef671aeb7dcd8700 Mon Sep 17 00:00:00 2001 From: Ramon Date: Sun, 20 Sep 2020 21:37:08 +0200 Subject: [PATCH 07/10] Changed the .coveragerc. --- .coveragerc | 2 ++ setup.cfg | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.coveragerc b/.coveragerc index 7981e20..9817e4b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,3 +6,5 @@ omit = */docs/* */test_resources/* tests/* +include = + barentsz/* diff --git a/setup.cfg b/setup.cfg index 922608f..4caafa0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,9 +11,6 @@ disable = no-value-for-parameter, isinstance-second-argument-not-valid-type, -[coverage:run] -include = barentsz/* - [isort] multi_line_output = 3 include_trailing_comma = True From 529b74b299843b8a43dba7b4bb1af70e8d0b291c Mon Sep 17 00:00:00 2001 From: Ramon Date: Sun, 20 Sep 2020 22:19:45 +0200 Subject: [PATCH 08/10] Fixed linting and mypy warnings. --- barentsz/_discover.py | 22 ++++++++++++---------- barentsz/_typings.py | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/barentsz/_discover.py b/barentsz/_discover.py index 4671539..66f9f83 100644 --- a/barentsz/_discover.py +++ b/barentsz/_discover.py @@ -1,8 +1,8 @@ import glob -import inspect import re import sys from importlib import import_module +from inspect import getmembers, isfunction, ismethod, isclass from pathlib import Path from typing import ( Any, @@ -25,8 +25,8 @@ ) from barentsz._attribute import Attribute -from barentsz._typings import exclude_pred from barentsz._here import here +from barentsz._typings import ClsPredicate def discover( @@ -166,7 +166,8 @@ def discover_classes( include_privates: bool = False, in_private_modules: bool = False, raise_on_fail: bool = False, - exclude: Union[type, exclude_pred, Iterable[Union[type, exclude_pred]]] = None + exclude: Union[type, ClsPredicate, + Iterable[Union[type, ClsPredicate]]] = None ) -> List[type]: """ Discover any classes within the given source and according to the given @@ -186,14 +187,15 @@ def discover_classes( """ exclude_ = _ensure_set(exclude) - elements = _discover_elements(source, inspect.isclass, include_privates, + elements = _discover_elements(source, isclass, include_privates, in_private_modules, raise_on_fail) result = list({cls for cls in elements if (signature is Any or subclass_of(cls, signature)) and cls not in exclude_}) - exclude_predicates = [e for e in exclude_ if inspect.isfunction(e)] + + exclude_predicates = (e for e in exclude_ if isfunction(e)) for pred in exclude_predicates: - result = [cls for cls in result if not pred(cls)] + result = [cls for cls in result if not pred(cls)] # type: ignore[operator] # noqa result.sort(key=lambda cls: cls.__name__) return result @@ -222,11 +224,11 @@ def discover_functions( """ def filter_(*args_: Iterable[Any]) -> bool: - return (inspect.isfunction(*args_) - or inspect.ismethod(*args_)) + return (isfunction(*args_) + or ismethod(*args_)) if not isinstance(source, type): - filter_ = inspect.isfunction # type: ignore + filter_ = isfunction # type: ignore elements = _discover_elements(source, filter_, include_privates, in_private_modules, raise_on_fail) @@ -329,7 +331,7 @@ def _discover_elements( raise_on_fail) elements = [elem for src in sources - for _, elem in inspect.getmembers(src, filter_) + for _, elem in getmembers(src, filter_) if (in_private_modules or not src.__name__.startswith('_')) and (include_privates or not elem.__name__.startswith('_'))] return elements diff --git a/barentsz/_typings.py b/barentsz/_typings.py index 454d94b..236c008 100644 --- a/barentsz/_typings.py +++ b/barentsz/_typings.py @@ -1,3 +1,3 @@ from typing import Callable -exclude_pred = Callable[[type], bool] +ClsPredicate = Callable[[type], bool] From 75b8a0b8f659c4034adf055e5989d819235e716a Mon Sep 17 00:00:00 2001 From: Ramon Date: Sun, 20 Sep 2020 22:21:40 +0200 Subject: [PATCH 09/10] Import formatting. --- barentsz/_discover.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/barentsz/_discover.py b/barentsz/_discover.py index 66f9f83..bd85496 100644 --- a/barentsz/_discover.py +++ b/barentsz/_discover.py @@ -2,7 +2,12 @@ import re import sys from importlib import import_module -from inspect import getmembers, isfunction, ismethod, isclass +from inspect import ( + getmembers, + isclass, + isfunction, + ismethod, +) from pathlib import Path from typing import ( Any, From aef79795c1869c94e721e7825e4f0e9ca80a5e53 Mon Sep 17 00:00:00 2001 From: Ramon Date: Sun, 20 Sep 2020 22:25:06 +0200 Subject: [PATCH 10/10] More logging on complexity per module. --- SConstruct.py | 1 + 1 file changed, 1 insertion(+) diff --git a/SConstruct.py b/SConstruct.py index d939c98..663d37b 100644 --- a/SConstruct.py +++ b/SConstruct.py @@ -47,6 +47,7 @@ def _exec(cmd: str) -> None: _exec(f'mypy {_SUBJECT} --show-error-codes --disallow-untyped-defs') if 'complexity' in COMMAND_LINE_TARGETS: + _exec(f'radon cc {_SUBJECT}') _exec(f'radon cc {_SUBJECT} -nc --total-average') _exec(f'xenon {_SUBJECT} --max-absolute B --max-modules A --max-average A')