Merge pull request #60 from demml/python-3.12

.github/workflows/examples.yml

@@ -38,7 +38,7 @@ jobs:
           rm -rf /opt/hostedtoolcache/Ruby
           rm -rf /opt/hostedtoolcache/go
           rm -rf /opt/hostedtoolcache/node
-          make setup.project
+          make setup.install
 
       - name: Run Sklearn Basic example
         run: uv run python examples/sklearn/basic.py && rm -rf opsml_registries && rm opsml.db

.github/workflows/integration.yml

@@ -62,7 +62,7 @@ jobs:
       - name: Set up Python ${{ matrix.python-version }}
         run: uv python install ${{ matrix.python-version }}
 
-      - run: make setup.project
+      - run: make setup.install
       - run: make test.integration.aws
 
   AZURE-Integration:
@@ -119,7 +119,7 @@ jobs:
       - name: Set up Python ${{ matrix.python-version }}
         run: uv python install ${{ matrix.python-version }}
 
-      - run: make setup.project
+      - run: make setup.install
       - run: make test.integration.azure
 
   GCS-Integration:
@@ -182,5 +182,5 @@ jobs:
         run: uv python install ${{ matrix.python-version }}
 
       - run: |
-          make setup.project
+          make setup.install
           make test.integration.gcp

.github/workflows/lint-unit-tests.yml

@@ -48,14 +48,14 @@ jobs:
       - name: Set up Python ${{ matrix.python-version }}
         run: uv python install ${{ matrix.python-version }}
 
-      - run: make setup.project
+      - run: make setup.install
       - run: make lints.ci
 
   Unit-Test-Ubuntu:
     needs: Lints
     strategy:
       matrix:
-        python-version: ["3.9", "3.11"]
+        python-version: ["3.10", "3.12"]
         poetry-version: ["1.7.1"]
         os: [ubuntu]
         node-version: ["22.5.1"]
@@ -101,14 +101,14 @@ jobs:
           rm -rf /opt/hostedtoolcache/Ruby
           rm -rf /opt/hostedtoolcache/go
           rm -rf /opt/hostedtoolcache/node
-          make setup.project
+          make setup.install
           make test.unit
 
   Unit-Test-MacOS:
     needs: Lints
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.11"]
         poetry-version: ["1.7.1"]
         os: [macos]
         node-version: ["22.5.1"]
@@ -164,15 +164,15 @@ jobs:
           rm -rf /opt/hostedtoolcache/Ruby
           rm -rf /opt/hostedtoolcache/go
           rm -rf /opt/hostedtoolcache/node
-          make setup.project
+          make setup.install
 
       - name: Unit Tests
         run: make test.unit
 
   Unit-Test-Coverage:
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.11"]
         poetry-version: ["1.7.1"]
         os: [ubuntu]
         node-version: ["22.5.1"]
@@ -220,7 +220,7 @@ jobs:
           rm -rf /opt/hostedtoolcache/go
           rm -rf /opt/hostedtoolcache/node
           sudo apt clean
-          make setup.project
+          make setup.install
           make test.coverage
 
       - name: Upload coverage reports to Codecov
@@ -309,7 +309,7 @@ jobs:
       - name: Set up Python ${{ matrix.python-version }}
         run: uv python install ${{ matrix.python-version }}
 
-      - run: make setup.project
+      - run: make setup.install
       - run: make test.registry
         env:
           OPSML_TRACKING_URI: postgresql://postgres:postgres@localhost:5432/postgres?connect_timeout=5&sslmode=disable
@@ -353,7 +353,7 @@ jobs:
       - name: Set up Python ${{ matrix.python-version }}
         run: uv python install ${{ matrix.python-version }}
 
-      - run: make setup.project
+      - run: make setup.install
       - run: make test.registry
         env:
           OPSML_TRACKING_URI: mysql+pymysql://[email protected]:3306/mysql?connect_timeout=5

.github/workflows/publish-docs.yml

@@ -24,7 +24,7 @@ jobs:
         run: uv python install 3.11
 
       - name: Install dependencies
-        run: make setup.project
+        run: make setup.install
 
       - name: Publish Docs
         run: make publish.docs

.github/workflows/release-candidate.yml

@@ -29,11 +29,11 @@ jobs:
           sudo swapoff -a
           sudo rm -f /swapfile
           sudo apt clean
-          make setup.project
+          make setup.install
           pip install -U twine
           make lints.ci
           make test.unit
-          make setup.project
+          make setup.install
           if [[ "v$(poetry version --short)" == *"rc"* ]]; 
           then
             poetry build

.github/workflows/release.yml

@@ -38,11 +38,10 @@ jobs:
           rm -rf /opt/hostedtoolcache/Ruby
           rm -rf /opt/hostedtoolcache/go
           rm -rf /opt/hostedtoolcache/node
-          make setup.project
+          make setup.install
           pip install -U twine
           make lints.ci
           make test.unit
-          make setup.project
           test "v$(poetry version --short)" "=" "$GITHUB_REF_NAME"
           poetry build
           twine check --strict dist/*

Makefile

@@ -47,11 +47,15 @@ lints: lints.format_check lints.ruff lints.pylint lints.mypy lints.gitleaks
 lints.ci: lints.format_check lints.ruff lints.pylint lints.mypy
 
 .PHONY: test
-setup: setup.sysdeps setup.python setup.project
-
-.PHONY: setup.project
-setup.project:
-	uv sync --all-extras --dev
+setup: setup.sysdeps setup.python setup.install
+
+.PHONY: setup.install
+setup.install:
+	uv sync --all-extras  \
+		--group lints \
+		--group test \
+		--group machine-learning \
+		--group torch-deps 
 
 .PHONY: test.unit
 test.unit:

opsml/model/interfaces/huggingface.py

@@ -309,7 +309,7 @@ def convert_to_onnx(self, **kwargs: Path) -> None:
             Converted model or pipeline is accessible via the `onnx_model` attribute.
             """
 
-            logger.info("Staring conversion of HuggingFace model to ONNX")
+            logger.info("Starting conversion of HuggingFace model to ONNX")
 
             assert (
                 self.onnx_args is not None

opsml/model/onnx/torch_converter.py

@@ -7,7 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 
 from pathlib import Path
-from typing import List, Tuple, Union, cast
+from typing import Any, List, Tuple, Union, cast
 
 import onnx
 import onnxruntime as rt
@@ -83,12 +83,14 @@ def convert_to_onnx(self, path: Path) -> OnnxModel:
         onnx_args = self._get_additional_model_args()
         # export
         self.interface.model.eval()  # force model into evaluation mode
+        logger.debug(f"Exporting model to ONNX: {path}")
         torch.onnx.export(
             model=self.interface.model,
-            args=arg_data,
+            args=cast(tuple[Any, ...], arg_data),
             f=path.as_posix(),
             **onnx_args.model_dump(exclude={"options"}),
         )
+        logger.debug(f"Model exported to ONNX: {path}")
 
         # load
         return OnnxModel(

pyproject.toml

@@ -3,7 +3,7 @@ name = "opsml"
 version = "3.0.0"
 description = "Add your description here"
 readme = "README.md"
-requires-python = '>=3.9, <=3.12'
+requires-python = '>=3.9, <=3.13'
 dependencies = [
     "fsspec>=2023.1.0, <=2024.2.0",
     "joblib~=1.3",
@@ -81,12 +81,14 @@ azure = ["adlfs~=2024.2.0"]
 postgres = ["psycopg2~=2.9.6"]
 mysql = ["pymysql~=1.0.2"]
 onnx = [
-    "onnx~=1.16.0; platform_system == 'Linux'",
-    "onnx==1.16.1; platform_system == 'Windows'",
-    "onnx~=1.16.0; platform_system == 'Darwin'"
+    "onnx~=1.16.0; platform_system == 'Linux' and python_version <= '3.11'",
+    "onnx==1.16.1; platform_system == 'Windows' and python_version <= '3.11'",
+    "onnx~=1.16.0; platform_system == 'Darwin' and python_version <= '3.11'",
+    "onnx>=1.17.0; python_version >= '3.12'",
 ]
 onnxruntime = [
-    "onnxruntime~=1.16.0",
+    "onnxruntime~=1.16.0; python_version <= '3.11'",
+    "onnxruntime>=1.18.0; python_version >= '3.12'",
 ]
 tf_onnx = [
     "tf2onnx~=1.16.1",
@@ -113,40 +115,49 @@ all = [
     "opsml[torch_onnx]",
 ]
 
-[tool.uv]
-dev-dependencies = [
+[dependency-groups]
+lints = [
     "ruff>=0.6.2",
+    "types-pymysql~=1.0.19.1",
+    "types-requests~=2.28.11.7",
+    "black~=24.3.0",
+    "mypy~=1.0",
+    "types-protobuf~=4.21.0.3",
+    "pylint~=2.17.5",
+    "pylint-pydantic~=0.2.4",
+    "types-pyyaml~=6.0.12.12",
+    "isort~=5.12.0",  
+]
+test = [
     "pytest~=7.1.2",
     "pytest-lazy-fixture~=0.6.3",
     "pytest-cov~=3.0.0",
     "coverage[toml]~=6.5.0",
-    "torch~=2.1.0",
-    "torchvision~=0.16",
+    "pytest-rerunfailures~=13.0",
+]
+machine-learning = [
     "lightgbm>=4.0.0, <4.4.0",
     "xgboost~=1.7.1",
     "sentence-transformers~=3.0.1",
     "optimum>=1.21.4",
-    "tensorflow~=2.16.0; platform_system == 'Linux'",
-    "tf-keras==2.16.0; platform_system == 'Linux'",
-    "transformers>=4.41.0, <4.42.0",
-    "lightning~=2.1.2",
-    "pytest-rerunfailures~=13.0",
+    "tensorflow>=2.16.0; platform_system == 'Linux'",
+    "tf-keras>=2.16.0; platform_system == 'Linux'",
     "catboost~=1.2.2",
-    "accelerate~=0.26.1",
-    "vowpalwabbit~=9.9.0; platform_system == 'Linux' and python_version < '3.11'",
+    "vowpalwabbit~=9.10.0; platform_system == 'Linux' and python_version < '3.11'",
     "matplotlib <3.9.1; platform_system == 'Windows'",
-    "types-pymysql~=1.0.19.1",
-    "types-requests~=2.28.11.7",
-    "black~=24.3.0",
-    "mypy~=1.0",
-    "types-protobuf~=4.21.0.3",
-    "pylint~=2.17.5",
-    "pylint-pydantic~=0.2.4",
-    "types-pyyaml~=6.0.12.12",
-    "isort~=5.12.0",
     "mkdocs-material~=9.5",
     "mkdocstrings[python]~=0.25.0",
     "confluent-kafka>=2.5.3",
+    "transformers>=4.42.0",
+    "accelerate>=1.0.1",
+]
+torch-deps =  [
+    "torch~=2.1.0; python_version <= '3.11'",
+    "torchvision~=0.16; python_version <= '3.11'",
+    "lightning~=2.1.2; python_version <= '3.11'",
+    "torch==2.2.0; python_version > '3.11'",
+    "torchvision>=0.16.2; python_version > '3.11'",
+    "lightning>=2.4.0; python_version > '3.11'",
 ]
 
 [tool.uv.sources]

tests/conftest.py

@@ -875,7 +875,7 @@ def multi_input_tf_example():
 def pytorch_resnet() -> TorchModel:
     import torch
 
-    loaded_model = torch.load("tests/assets/resnet.pt")
+    loaded_model = torch.load("tests/assets/resnet18.pt")
     data = torch.randn(1, 3, 224, 224)
 
     return TorchModel(model=loaded_model, sample_data=data)
@@ -2403,22 +2403,15 @@ def voting_regressor(example_dataframe):
 
 
 @pytest.fixture(scope="module")
-def deeplabv3_resnet50():
-    import torch
-    from PIL import Image
+def squeezenet():
     from torchvision import transforms
 
-    model = torch.hub.load(
-        "pytorch/vision:v0.8.0",
-        "deeplabv3_resnet50",
-        pretrained=True,
-        skip_validation=True,
-    )
-    model.eval()
-
+    model = torch.load("tests/assets/squeeze.pt")
     input_image = Image.open("tests/assets/deeplab.jpg")
     preprocess = transforms.Compose(
         [
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
             transforms.ToTensor(),
             transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
         ]

tests/test_interface/test_model_interface.py

@@ -31,10 +31,10 @@ def test_tf_interface(tf_transformer_example: TensorFlowModel) -> None:
 
 
 @pytest.mark.flaky(reruns=2, reruns_delay=5)
-def test_torch_interface(deeplabv3_resnet50: TorchModel) -> None:
-    assert deeplabv3_resnet50.model_type == "DeepLabV3"
-    prediction = deeplabv3_resnet50.get_sample_prediction()
-    assert prediction.prediction_type == "collections.OrderedDict"
+def test_torch_interface(squeezenet: TorchModel) -> None:
+    assert squeezenet.model_type == "SqueezeNet"
+    prediction = squeezenet.get_sample_prediction()
+    assert prediction.prediction_type == "torch.Tensor"
 
 
 @pytest.mark.flaky(reruns=1, reruns_delay=2)

tests/test_model/test_model_data.py

@@ -11,7 +11,7 @@
 # most methods are tested as part of other unit tests
 
 
-def test_model_data_base(numpy_data: NumpyData):
+def test_model_data_base(numpy_data: NumpyData) -> None:
     model_data = ModelDataHelper(input_data=numpy_data.data, data_type=AllowedDataType.NUMPY.value)
 
     with pytest.raises(NotImplementedError):
@@ -27,7 +27,7 @@ def test_model_data_base(numpy_data: NumpyData):
         model_data.feature_dict
 
 
-def test_dataframe(pandas_data: PandasData):
+def test_dataframe(pandas_data: PandasData) -> None:
     """Test ModelData for pandas dataframe"""
     pd_data = get_model_data(input_data=pandas_data.data, data_type=AllowedDataType.PANDAS)
 
@@ -37,7 +37,7 @@ def test_dataframe(pandas_data: PandasData):
     assert isinstance(pd_data.feature_types, zip)
 
 
-def test_numpy(numpy_data: NumpyData):
+def test_numpy(numpy_data: NumpyData) -> None:
     """Test ModelData for numpy array"""
     numpy_data = get_model_data(
         input_data=numpy_data.data,
@@ -54,7 +54,7 @@ def test_numpy(numpy_data: NumpyData):
         numpy_data.convert_dataframe_column(column_type="test", convert_column_type="test")
 
 
-def test_dictionary(numpy_data: NumpyData):
+def test_dictionary(numpy_data: NumpyData) -> None:
     """Test ModelData for dictionary"""
     data = {"test": numpy_data.data}
     dict_data = get_model_data(