Scouter integration

.gitignore

@@ -125,6 +125,7 @@ notebooks/
 .test_local.py
 lightning_logs/
 node_modules/
+target/
 
 # git
 **/*.orig

opsml/app/routes/drift.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2024-current Demml, Inc.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# pylint: disable=protected-access
+
+from fastapi import APIRouter, HTTPException, Request, status
+
+from opsml.app.routes.pydantic_models import DriftProfileRequest, Success
+from opsml.helpers.logging import ArtifactLogger
+from opsml.registry.sql.base.server import ServerModelCardRegistry
+
+logger = ArtifactLogger.get_logger()
+
+router = APIRouter()
+
+
+@router.post("/drift/profile", name="metric_put", response_model=Success)
+def insert_metric(request: Request, payload: DriftProfileRequest) -> Success:
+    """Uploads drift profile to scouter-server
+
+    Args:
+        request:
+            FastAPI request object
+        payload:
+            DriftProfileRequest
+
+    Returns:
+        200
+    """
+
+    model_reg: ServerModelCardRegistry = request.app.state.registries.model._registry
+
+    try:
+        model_reg.insert_drift_profile(payload.profile)
+        return Success()
+    except Exception as error:
+        logger.error(f"Failed to insert metrics: {error}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to insert drift profile"
+        ) from error

opsml/app/routes/pydantic_models.py

@@ -517,3 +517,7 @@ class SecurityQuestionResponse(BaseModel):
 class TempRequest(BaseModel):
     username: str
     answer: str
+
+
+class DriftProfileRequest(BaseModel):
+    profile: str

opsml/app/routes/router.py

@@ -11,6 +11,7 @@
     auth,
     cards,
     data,
+    drift,
     files,
     healthcheck,
     metrics,
@@ -37,6 +38,7 @@ def build_router(dependencies: Optional[Sequence[Any]] = None) -> APIRouter:
     api_router.include_router(parameters.router, tags=["parameters"], prefix="/opsml", dependencies=dependencies)
     api_router.include_router(runs.router, tags=["runs"], prefix="/opsml", dependencies=dependencies)
     api_router.include_router(ui.router, tags=["ui"], dependencies=dependencies)
+    api_router.include_router(drift.router, tags=["drift"], prefix="/opsml", dependencies=dependencies)
     api_router.include_router(auth.router, tags=["auth"], prefix="/opsml")
 
     return api_router

opsml/data/interfaces/_base.py

@@ -144,7 +144,7 @@ def load_data_profile(self, path: Path) -> None:
                 Pathlib object
         """
 
-        profile = DataProfile.load_from_json(path.read_text(encoding="utf-8"))
+        profile = DataProfile.model_validate_json(path.read_text(encoding="utf-8"))
         self.data_profile = profile
 
     def save_data_profile(self, path: Path) -> None:
@@ -170,7 +170,7 @@ def create_data_profile(self, bin_size: int = 20, features: Optional[List[str]]
         profiler = DataProfiler()
 
         if self.data_profile is None:
-            self.data_profile = profiler.create_profile_report(self.data, bin_size, features)
+            self.data_profile = profiler.create_profile_report(self.data, bin_size)
             return self.data_profile
 
         logger.info("Data profile already exists")

opsml/model/interfaces/base.py

@@ -1,15 +1,17 @@
 from dataclasses import dataclass
 from functools import cached_property
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, cast
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
 from uuid import UUID
 
 import joblib
 import numpy as np
 import pandas as pd
 import polars as pl
 import pyarrow as pa
+from numpy.typing import NDArray
 from pydantic import BaseModel, ConfigDict, field_validator, model_validator
+from scouter import DriftConfig, Drifter, DriftProfile
 
 from opsml.data import DataInterface
 from opsml.helpers.utils import get_class_name
@@ -115,6 +117,7 @@
     modelcard_uid: str = ""
     feature_map: Dict[str, Feature] = {}
     sample_data_interface_type: str = CommonKwargs.UNDEFINED.value
+    drift_profile: Optional[DriftProfile] = None
 
     model_config = ConfigDict(
         protected_namespaces=("protect_",),
@@ -341,6 +344,55 @@
 
         return self.sample_data
 
+    def create_drift_profile(
+        self,
+        data: Union[pl.DataFrame, pd.DataFrame, NDArray[Any], pa.Table],
+        drift_config: DriftConfig,
+    ) -> DriftProfile:
+        """Create a drift profile from data to use for model monitoring.
+
+        Args:
+            data:
+                Data to create a monitoring profile from. Data can be a numpy array, pyarrow table,
+                a polars dataframe or pandas dataframe. Data is expected to not contain
+                any missing values, NaNs or infinities and it typically the data used for training a model.
+            drift_config:
+                Configuration for the monitoring profile.
+
+        """
+
+        if self.drift_profile is not None:
+            return self.drift_profile
+
+        drifter = Drifter()
+        profile = drifter.create_drift_profile(
+            data=data,
+            drift_config=drift_config,
+        )
+        self.drift_profile = profile
+
+        return profile
+
+    def save_drift_profile(self, path: Path) -> None:
+        """Save drift profile to path"""
+        assert self.drift_profile is not None, "No drift profile detected in interface"
+        self.drift_profile.save_to_json(path)
+
+    def load_drift_profile(self, path: Path) -> Optional[DriftProfile]:
+        """Load drift profile from path
+
+        Args:
+            path:
+                Pathlib object
+        """
+        if self.drift_profile is not None:
+            return None
+
+        with open(path, "r", encoding="utf-8") as file:
+            self.drift_profile = DriftProfile.model_validate_json(file.read())
+
+        return self.drift_profile
+
     @staticmethod
     def name() -> str:
         return ModelInterface.__name__

opsml/profile/profile_data.py

@@ -4,7 +4,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import List, Optional, Union
+from typing import Union
 
 import pandas as pd
 import polars as pl
@@ -16,7 +16,6 @@
     def create_profile_report(
         data: Union[pd.DataFrame, pl.DataFrame],
         bin_size: int = 20,
-        features: Optional[List[str]] = None,
     ) -> DataProfile:
         """
         Creates a `scouter` data profile report
@@ -26,15 +25,13 @@
                 data to profile
             bin_size:
                 number of bins for histograms. Default is 20
-            features:
-                Optional list of features to profile
 
         Returns:
             `DataProfile`
         """
         profiler = Profiler()
 
-        return profiler.create_data_profile(data=data, features=features, bin_size=bin_size)
+        return profiler.create_data_profile(data=data, bin_size=bin_size)
 
     @staticmethod
     def load_profile(data: str) -> DataProfile:
@@ -48,4 +45,4 @@
             `DataProfile`
         """
 
-        return DataProfile.load_from_json(data)
+        return DataProfile.model_validate_json(data)

opsml/registry/sql/base/client.py

@@ -19,6 +19,7 @@
 from opsml.registry.semver import CardVersion, VersionType
 from opsml.registry.sql.base.registry_base import SQLRegistryBase
 from opsml.registry.sql.base.utils import log_card_change
+from opsml.settings.config import config
 from opsml.storage.api import RequestType, api_routes
 from opsml.storage.client import ApiStorageClient, StorageClient
 from opsml.types import RegistryType
@@ -258,6 +259,13 @@
         if not exists:
             raise ValueError("ModelCard must be associated with a valid DataCard uid")
 
+    def insert_drift_profile(self, drift_profile: str) -> None:
+        self._session.request(
+            route=api_routes.DRIFT_PROFILE,
+            request_type=RequestType.POST,
+            json={"profile": drift_profile},
+        )
+
     def register_card(
         self,
         card: Card,
@@ -292,18 +300,18 @@
             )
 
         else:
-            model_card = cast(ModelCard, card)
+            card = cast(ModelCard, card)
 
-            if model_card.to_onnx:
+            if card.to_onnx:
                 if not check_package_exists("onnx"):
                     raise ModuleNotFoundError(
                         """To convert a model to onnx, please install onnx via one of the extras
                         (opsml[sklearn_onnx], opsml[tf_onnx], opsml[torch_onnx]) or set to_onnx to False.
                         """
                     )
 
-            if model_card.datacard_uid is not None:
-                self._validate_datacard_uid(uid=model_card.datacard_uid)
+            if card.datacard_uid is not None:
+                self._validate_datacard_uid(uid=card.datacard_uid)
 
             super().register_card(
                 card=card,
@@ -312,6 +320,13 @@
                 build_tag=build_tag,
             )
 
+            # write profile to scouter
+            if card.interface.drift_profile is not None and config.scouter_server_uri is not None:
+                try:
+                    self.insert_drift_profile(drift_profile=card.interface.drift_profile.model_dump_json())
+                except Exception as exc:  # pylint: disable=broad-except
+                    logger.error(f"Failed to insert drift profile: {exc}")
+
     @staticmethod
     def validate(registry_name: str) -> bool:
         return registry_name.lower() == RegistryType.MODEL.value

opsml/registry/sql/base/query_engine.py

@@ -506,6 +506,7 @@ def query_page(
             Tuple of card summary
         """
 
+        ## build versions
         versions = select(
             table.repository,
             table.name,
@@ -546,6 +547,9 @@ def query_page(
                 ),
             )
 
+        versions = versions.subquery()
+        stats = stats.subquery()
+
         filtered_versions = (
             select(
                 versions.c.repository,

opsml/registry/sql/base/server.py

@@ -35,6 +35,8 @@
 from opsml.registry.sql.connectors.connector import DefaultConnector
 from opsml.settings.config import config
 from opsml.storage.client import StorageClient
+from opsml.storage.scouter import SCOUTER_CLIENT as scouter_client
+from opsml.storage.scouter import ScouterClient
 from opsml.types import RegistryTableNames, RegistryType
 from opsml.types.extra import Message, User
 
@@ -59,6 +61,7 @@
 
         self.engine = get_query_engine(db_engine=db_initializer.engine, registry_type=registry_type)
         self._table = SQLTableGetter.get_table(table_name=self.table_name)
+        self._scouter_client = scouter_client
 
     @property
     def registry_type(self) -> RegistryType:
@@ -69,11 +72,26 @@
     def validate(registry_name: str) -> bool:
         raise NotImplementedError
 
+    @property
+    def scouter_client(self) -> Optional[ScouterClient]:
+        return self._scouter_client
+
     @property
     def unique_repositories(self) -> Sequence[str]:
         """Returns a list of unique repositories"""
         return self.engine.get_unique_repositories(table=self._table)
 
+    def insert_drift_profile(self, drift_profile: str) -> None:
+        """Insert drift profile into scouter server
+
+        Args:
+            drift_profile:
+                drift profile
+        """
+
+        if self.scouter_client is not None:
+            self.scouter_client.insert_drift_profile(drift_profile=drift_profile)
+
     def query_stats(self, search_term: Optional[str] = None) -> Dict[str, int]:
         """Query stats from Card Database
         Args:
@@ -356,18 +374,18 @@
             )
 
         else:
-            model_card = cast(ModelCard, card)
+            card = cast(ModelCard, card)
 
-            if model_card.to_onnx:
+            if card.to_onnx:
                 if not check_package_exists("onnx"):
                     raise ModuleNotFoundError(
                         """To convert a model to onnx, please install onnx via one of the extras
                         (opsml[sklearn_onnx], opsml[tf_onnx], opsml[torch_onnx]) or set to_onnx to False.
                         """
                     )
 
-            if model_card.datacard_uid is not None:
-                self._validate_datacard_uid(uid=model_card.datacard_uid)
+            if card.datacard_uid is not None:
+                self._validate_datacard_uid(uid=card.datacard_uid)
 
             super().register_card(
                 card=card,
@@ -376,6 +394,17 @@
                 build_tag=build_tag,
             )
 
+            print(config.scouter_server_uri)
+
+            # write profile to scouter
+            if card.interface.drift_profile is not None and config.scouter_server_uri is not None:
+                try:
+                    self.insert_drift_profile(
+                        drift_profile=card.interface.drift_profile.model_dump_json(),
+                    )
+                except Exception as exc:  # pylint: disable=broad-except
+                    logger.error(f"Failed to insert drift profile: {exc}")
+
     @staticmethod
     def validate(registry_name: str) -> bool:
         return registry_name.lower() == RegistryType.MODEL.value

opsml/settings/config.py

@@ -25,6 +25,7 @@ class OpsmlConfig(BaseSettings):
     opsml_prod_token: str = "staging"
     opsml_proxy_root: str = "opsml-root:/"
     opsml_registry_path: str = "model_registry"
+    opsml_client_path_prefix: str = "opsml"
     opsml_testing: bool = bool(0)
     download_chunk_size: int = 31457280  # 30MB
     upload_chunk_size: int = 31457280  # 30MB
@@ -40,6 +41,13 @@ class OpsmlConfig(BaseSettings):
     opsml_username: Optional[str] = None
     opsml_password: Optional[str] = None
 
+    # scouter settings
+    scouter_server_uri: Optional[str] = None
+    scouter_username: Optional[str] = None
+    scouter_password: Optional[str] = None
+    scouter_path_prefix: str = "scouter"
+    scouter_auth: bool = False
+
     # Auth
     opsml_auth: bool = False