diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5e4be400..170abea9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24e816d9..77a3f9a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,10 +5,11 @@ on: [push, pull_request, workflow_call] jobs: build: strategy: + fail-fast: false max-parallel: 20 matrix: os: [ubuntu-latest, macos-14, windows-latest] - python-version: ["3.9", "3.x"] + python-version: ["3.9", "3.12"] runs-on: ${{ matrix.os }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a539f1a..e3293925 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: rev: v2.3.0 hooks: - id: codespell - stages: [Nonepre-commitNone, commit-msg] + stages: [pre-commit, commit-msg] exclude_types: [html] additional_dependencies: [tomli] # needed to read pyproject.toml below py3.11 diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 9ecbb76b..94d13ee6 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -224,7 +224,7 @@ GEM rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) - rexml (3.3.2) + rexml (3.3.6) strscan rouge (3.26.0) ruby2_keywords (0.0.5) @@ -251,7 +251,7 @@ GEM unf_ext unf_ext (0.0.8.2) unicode-display_width (1.8.0) - webrick (1.8.1) + webrick (1.8.2) PLATFORMS arm64-darwin-22 diff --git a/pyproject.toml b/pyproject.toml index b0b8683a..f472887e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,35 +19,41 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - + "ruamel.yaml", + "numpy<2.0.0", ] version = "2024.7.30" [project.optional-dependencies] ci = [ - "pytest>=8", - "pytest-cov>=4", - "coverage", - "numpy<2.0.0", - "ruamel.yaml", - "msgpack", - "tqdm", - "pymongo", - "pandas", - "pint", - "orjson", - "types-orjson", - "types-requests", - "torch" + "coverage", + "monty[optional]", + "pytest>=8", + "pytest-cov>=4", + "types-requests", ] +# dev is for "dev" module, not for development +dev = ["ipython"] docs = [ "sphinx", "sphinx_rtd_theme", ] +json = [ + "bson", + "orjson>=3.6.1", + "pandas", + "pydantic", + "pint", + "torch", +] +multiprocessing = ["tqdm"] +optional = ["monty[dev,json,multiprocessing,serialization]"] +serialization = ["msgpack"] +task = ["requests", "invoke"] [tool.setuptools.packages.find] where = ["src"] -include = ["monty"] +include = ["monty", "monty.*"] [tool.black] line-length = 120 @@ -101,3 +107,4 @@ lint.select = [ ] lint.isort.required-imports = ["from __future__ import annotations"] +lint.isort.known-first-party = ["monty"] diff --git a/src/monty/dev.py b/src/monty/dev.py index 353674e4..cf468bce 100644 --- a/src/monty/dev.py +++ b/src/monty/dev.py @@ -231,9 +231,8 @@ def install_excepthook(hook_type: str = "color", **kwargs) -> int: """ try: from IPython.core import ultratb # pylint: disable=import-outside-toplevel - except ImportError: - warnings.warn("Cannot install excepthook, IPyhon.core.ultratb not available") - return 1 + except ImportError as exc: + raise ImportError("Cannot install excepthook, IPython not installed") from exc # Select the hook. hook = dict( diff --git a/src/monty/io.py b/src/monty/io.py index f24a63dd..9ca0f2cb 100644 --- a/src/monty/io.py +++ b/src/monty/io.py @@ -9,11 +9,7 @@ import errno import gzip import io - -try: - import lzma -except ImportError: - lzma = None # type: ignore[assignment] +import lzma import mmap import os import subprocess @@ -49,7 +45,7 @@ def zopen(filename: Union[str, Path], *args, **kwargs) -> IO: return bz2.open(filename, *args, **kwargs) if ext in {".GZ", ".Z"}: return gzip.open(filename, *args, **kwargs) - if lzma is not None and ext in {".XZ", ".LZMA"}: + if ext in {".XZ", ".LZMA"}: return lzma.open(filename, *args, **kwargs) return open(filename, *args, **kwargs) diff --git a/src/monty/itertools.py b/src/monty/itertools.py index 5a112a03..c4ed76d5 100644 --- a/src/monty/itertools.py +++ b/src/monty/itertools.py @@ -7,10 +7,7 @@ import itertools from typing import TYPE_CHECKING -try: - import numpy as np -except ImportError: - np = None +import numpy as np if TYPE_CHECKING: from typing import Iterable diff --git a/src/monty/json.py b/src/monty/json.py index f9899097..d0626d3e 100644 --- a/src/monty/json.py +++ b/src/monty/json.py @@ -18,49 +18,30 @@ from importlib import import_module from inspect import getfullargspec from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING from uuid import UUID, uuid4 -try: - import numpy as np -except ImportError: - np = None - -try: - import pydantic -except ImportError: - pydantic = None +import numpy as np +from ruamel.yaml import YAML -try: - from pydantic_core import core_schema -except ImportError: - core_schema = None +if TYPE_CHECKING: + from typing import Any try: import bson except ImportError: bson = None -try: - from ruamel.yaml import YAML -except ImportError: - YAML = None - try: import orjson except ImportError: orjson = None -try: - import torch -except ImportError: - torch = None - __version__ = "3.0.0" -def _load_redirect(redirect_file): +def _load_redirect(redirect_file) -> dict: try: with open(redirect_file) as f: yaml = YAML() @@ -71,7 +52,7 @@ def _load_redirect(redirect_file): return {} # Convert the full paths to module/class - redirect_dict = defaultdict(dict) + redirect_dict: dict = defaultdict(dict) for old_path, new_path in d.items(): old_class = old_path.split(".")[-1] old_module = ".".join(old_path.split(".")[:-1]) @@ -87,7 +68,7 @@ def _load_redirect(redirect_file): return dict(redirect_dict) -def _check_type(obj, type_str) -> bool: +def _check_type(obj, type_str: tuple[str, ...] | str) -> bool: """Alternative to isinstance that avoids imports. Checks whether obj is an instance of the type defined by type_str. This @@ -121,7 +102,7 @@ class B(A): mro = type(obj).mro() except TypeError: return False - return any(o.__module__ + "." + o.__name__ == ts for o in mro for ts in type_str) + return any(f"{o.__module__}.{o.__name__}" == ts for o in mro for ts in type_str) class MSONable: @@ -338,8 +319,11 @@ def __get_pydantic_core_schema__(cls, source_type, handler): """ pydantic v2 core schema definition """ - if core_schema is None: - raise RuntimeError("Pydantic >= 2.0 is required for validation") + try: + from pydantic_core import core_schema + + except ImportError as exc: + raise RuntimeError("Pydantic >= 2.0 is required for validation") from exc s = core_schema.with_info_plain_validator_function(cls.validate_monty_v2) @@ -541,7 +525,7 @@ def _recursive_name_object_map_replacement(d, name_object_map): class MontyEncoder(json.JSONEncoder): """ A Json Encoder which supports the MSONable API, plus adds support for - numpy arrays, datetime objects, bson ObjectIds (requires bson). + NumPy arrays, datetime objects, bson ObjectIds (requires bson). Usage:: # Add it as a *cls* keyword when using json.dump json.dumps(object, cls=MontyEncoder) @@ -586,8 +570,8 @@ def default(self, o) -> dict: if isinstance(o, Path): return {"@module": "pathlib", "@class": "Path", "string": str(o)} - if torch is not None and isinstance(o, torch.Tensor): - # Support for Pytorch Tensors. + # Support for Pytorch Tensors + if _check_type(o, "torch.Tensor"): d: dict[str, Any] = { "@module": "torch", "@class": "Tensor", @@ -599,23 +583,23 @@ def default(self, o) -> dict: d["data"] = o.numpy().tolist() return d - if np is not None: - if isinstance(o, np.ndarray): - if str(o.dtype).startswith("complex"): - return { - "@module": "numpy", - "@class": "array", - "dtype": str(o.dtype), - "data": [o.real.tolist(), o.imag.tolist()], - } + if isinstance(o, np.ndarray): + if str(o.dtype).startswith("complex"): return { "@module": "numpy", "@class": "array", "dtype": str(o.dtype), - "data": o.tolist(), + "data": [o.real.tolist(), o.imag.tolist()], } - if isinstance(o, np.generic): - return o.item() + return { + "@module": "numpy", + "@class": "array", + "dtype": str(o.dtype), + "data": o.tolist(), + } + + if isinstance(o, np.generic): + return o.item() if _check_type(o, "pandas.core.frame.DataFrame"): return { @@ -660,7 +644,7 @@ def default(self, o) -> dict: raise AttributeError(e) try: - if pydantic is not None and isinstance(o, pydantic.BaseModel): + if _check_type(o, "pydantic.main.BaseModel"): d = o.model_dump() elif ( dataclasses is not None @@ -668,7 +652,7 @@ def default(self, o) -> dict: and dataclasses.is_dataclass(o) ): # This handles dataclasses that are not subclasses of MSONAble. - d = dataclasses.asdict(o) # type: ignore[call-overload] + d = dataclasses.asdict(o) # type: ignore[call-overload, arg-type] elif hasattr(o, "as_dict"): d = o.as_dict() elif isinstance(o, Enum): @@ -790,11 +774,18 @@ def process_decoded(self, d): return cls_.from_dict(data) if issubclass(cls_, Enum): return cls_(d["value"]) - if pydantic is not None and issubclass( - cls_, pydantic.BaseModel - ): # pylint: disable=E1101 - d = {k: self.process_decoded(v) for k, v in data.items()} - return cls_(**d) + + try: + import pydantic + + if issubclass(cls_, pydantic.BaseModel): + d = { + k: self.process_decoded(v) for k, v in data.items() + } + return cls_(**d) + except ImportError: + pass + if ( dataclasses is not None and (not issubclass(cls_, MSONable)) @@ -803,17 +794,23 @@ def process_decoded(self, d): d = {k: self.process_decoded(v) for k, v in data.items()} return cls_(**d) - elif torch is not None and modname == "torch" and classname == "Tensor": - if "Complex" in d["dtype"]: - return torch.tensor( # pylint: disable=E1101 - [ - np.array(r) + np.array(i) * 1j - for r, i in zip(*d["data"]) - ], - ).type(d["dtype"]) - return torch.tensor(d["data"]).type(d["dtype"]) # pylint: disable=E1101 + elif modname == "torch" and classname == "Tensor": + try: + import torch # import torch is very expensive - elif np is not None and modname == "numpy" and classname == "array": + if "Complex" in d["dtype"]: + return torch.tensor( + [ + np.array(r) + np.array(i) * 1j + for r, i in zip(*d["data"]) + ], + ).type(d["dtype"]) + return torch.tensor(d["data"]).type(d["dtype"]) + + except ImportError: + pass + + elif modname == "numpy" and classname == "array": if d["dtype"].startswith("complex"): return np.array( [ @@ -867,8 +864,8 @@ def decode(self, s): """ if orjson is not None: try: - d = orjson.loads(s) # pylint: disable=E1101 - except orjson.JSONDecodeError: # pylint: disable=E1101 + d = orjson.loads(s) + except orjson.JSONDecodeError: d = json.loads(s) else: d = json.loads(s) @@ -925,6 +922,7 @@ def jsanitize( or (bson is not None and isinstance(obj, bson.objectid.ObjectId)) ): return obj + if isinstance(obj, (list, tuple)): return [ jsanitize( @@ -936,7 +934,8 @@ def jsanitize( ) for i in obj ] - if np is not None and isinstance(obj, np.ndarray): + + if isinstance(obj, np.ndarray): try: return [ jsanitize( @@ -950,8 +949,10 @@ def jsanitize( ] except TypeError: return obj.tolist() - if np is not None and isinstance(obj, np.generic): + + if isinstance(obj, np.generic): return obj.item() + if _check_type( obj, ( @@ -961,6 +962,7 @@ def jsanitize( ), ): return obj.to_dict() + if isinstance(obj, dict): return { str(k): jsanitize( @@ -972,10 +974,13 @@ def jsanitize( ) for k, v in obj.items() } + if isinstance(obj, (int, float)): return obj + if obj is None: return None + if isinstance(obj, (pathlib.Path, datetime.datetime)): return str(obj) @@ -997,7 +1002,7 @@ def jsanitize( if isinstance(obj, str): return obj - if pydantic is not None and isinstance(obj, pydantic.BaseModel): # pylint: disable=E1101 + if _check_type(obj, "pydantic.main.BaseModel"): return jsanitize( MontyEncoder().default(obj), strict=strict, diff --git a/src/monty/multiprocessing.py b/src/monty/multiprocessing.py index 25511820..4b7c720f 100644 --- a/src/monty/multiprocessing.py +++ b/src/monty/multiprocessing.py @@ -9,8 +9,8 @@ try: from tqdm.autonotebook import tqdm -except ImportError: - tqdm = None +except ImportError as exc: + raise ImportError("tqdm must be installed for this function.") from exc def imap_tqdm(nprocs: int, func: Callable, iterable: Iterable, *args, **kwargs) -> list: @@ -28,8 +28,6 @@ def imap_tqdm(nprocs: int, func: Callable, iterable: Iterable, *args, **kwargs) Returns: Results of Pool.imap. """ - if tqdm is None: - raise ImportError("tqdm must be installed for this function.") data = [] with Pool(nprocs) as pool: try: diff --git a/src/monty/serialization.py b/src/monty/serialization.py index a17b9913..6732a674 100644 --- a/src/monty/serialization.py +++ b/src/monty/serialization.py @@ -9,10 +9,7 @@ import os from typing import TYPE_CHECKING -try: - from ruamel.yaml import YAML -except ImportError: - YAML = None # type: ignore[arg-type] +from ruamel.yaml import YAML from monty.io import zopen from monty.json import MontyDecoder, MontyEncoder diff --git a/tasks.py b/tasks.py index 921412ad..7432034a 100755 --- a/tasks.py +++ b/tasks.py @@ -11,6 +11,7 @@ import requests from invoke import task + from monty import __version__ as ver from monty.os import cd diff --git a/tests/test_collections.py b/tests/test_collections.py index 8057f7bc..58d2af9e 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -3,6 +3,7 @@ import os import pytest + from monty.collections import AttrDict, FrozenAttrDict, Namespace, frozendict, tree TEST_DIR = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_design_patterns.py b/tests/test_design_patterns.py index c37bc87d..67a7ffc2 100644 --- a/tests/test_design_patterns.py +++ b/tests/test_design_patterns.py @@ -6,6 +6,7 @@ from typing import Any import pytest + from monty.design_patterns import cached_class, singleton diff --git a/tests/test_dev.py b/tests/test_dev.py index 8c44d8ae..13bb211b 100644 --- a/tests/test_dev.py +++ b/tests/test_dev.py @@ -7,6 +7,7 @@ from unittest.mock import patch import pytest + from monty.dev import deprecated, install_excepthook, requires # Set all warnings to always be triggered. diff --git a/tests/test_fractions.py b/tests/test_fractions.py index 7d5f2770..230dcf5a 100644 --- a/tests/test_fractions.py +++ b/tests/test_fractions.py @@ -1,6 +1,7 @@ from __future__ import annotations import pytest + from monty.fractions import gcd, gcd_float, lcm diff --git a/tests/test_functools.py b/tests/test_functools.py index 8bf3fabf..120e6bd7 100644 --- a/tests/test_functools.py +++ b/tests/test_functools.py @@ -5,6 +5,7 @@ import unittest import pytest + from monty.functools import ( TimeoutError, lazy_property, diff --git a/tests/test_io.py b/tests/test_io.py index 9daa17be..052bc471 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest + from monty.io import ( FileLock, FileLockException, diff --git a/tests/test_json.py b/tests/test_json.py index 9c9e572c..e59e89d5 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -8,10 +8,19 @@ from enum import Enum from typing import Union -try: - import numpy as np -except ImportError: - np = None +import numpy as np +import pytest + +from monty.json import ( + MontyDecoder, + MontyEncoder, + MSONable, + _load_redirect, + jsanitize, + load, +) + +from . import __version__ as TESTS_VERSION try: import pandas as pd @@ -38,18 +47,6 @@ except ImportError: ObjectId = None -import pytest -from monty.json import ( - MontyDecoder, - MontyEncoder, - MSONable, - _load_redirect, - jsanitize, - load, -) - -from . import __version__ as tests_version - TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_files") @@ -320,7 +317,7 @@ def test_unsafe_hash(self): def test_version(self): obj = self.good_cls("Hello", "World", "Python") d = obj.as_dict() - assert d["@version"] == tests_version + assert d["@version"] == TESTS_VERSION def test_nested_to_from_dict(self): GMC = GoodMSONClass @@ -564,7 +561,6 @@ def test_nan(self): d = json.loads(djson) assert isinstance(d[0], float) - @pytest.mark.skipif(np is None, reason="numpy not present") def test_numpy(self): x = np.array([1, 2, 3], dtype="int64") with pytest.raises(TypeError): @@ -872,9 +868,7 @@ def test_jsanitize_pandas(self): clean = jsanitize(s) assert clean == s.to_dict() - @pytest.mark.skipif( - np is None or ObjectId is None, reason="numpy and bson not present" - ) + @pytest.mark.skipif(ObjectId is None, reason="bson not present") def test_jsanitize_numpy_bson(self): d = { "a": ["b", np.array([1, 2, 3])], diff --git a/tests/test_os.py b/tests/test_os.py index c6ca8d7c..af5b156f 100644 --- a/tests/test_os.py +++ b/tests/test_os.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest + from monty.os import cd, makedirs_p from monty.os.path import find_exts, zpath diff --git a/tests/test_serialization.py b/tests/test_serialization.py index d2e0cf9c..e7c853a5 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -7,14 +7,14 @@ import pytest +from monty.serialization import dumpfn, loadfn +from monty.tempfile import ScratchDir + try: import msgpack except ImportError: msgpack = None -from monty.serialization import dumpfn, loadfn -from monty.tempfile import ScratchDir - class TestSerial: @classmethod diff --git a/tests/test_shutil.py b/tests/test_shutil.py index 16cd9aa7..e506b4ae 100644 --- a/tests/test_shutil.py +++ b/tests/test_shutil.py @@ -9,6 +9,7 @@ from pathlib import Path import pytest + from monty.shutil import ( compress_dir, compress_file, diff --git a/tests/test_tempfile.py b/tests/test_tempfile.py index 4dfd3b17..088aec67 100644 --- a/tests/test_tempfile.py +++ b/tests/test_tempfile.py @@ -4,6 +4,7 @@ import shutil import pytest + from monty.tempfile import ScratchDir TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_files")