Fix model is ready even if there is no model (kserve#3275)

check empty model final.

Signed-off-by: HAO <[email protected]>
Co-authored-by: koshino17 <[email protected]>

python/huggingfaceserver/huggingfaceserver/__main__.py

@@ -252,7 +252,7 @@ def load_model():
         logging.configure_logging(args.log_config_file)
     try:
         model = load_model()
-        kserve.ModelServer().start([model] if model.ready else [])
+        kserve.ModelServer().start([model])
     except Exception as e:
         import sys
 

python/kserve/kserve/errors.py

@@ -132,3 +132,16 @@ async def not_implemented_error_handler(_, exc):
     return JSONResponse(
         status_code=HTTPStatus.NOT_IMPLEMENTED, content={"error": str(exc)}
     )
+
+
+class NoModelReady(RuntimeError):
+    def __init__(self, models: [], detail: str = None):
+        self.models = models
+        self.detail = detail
+
+    def __str__(self):
+        model_name_list = [model.name for model in self.models]
+        self.error_msg = f"Models with name {','.join(model_name_list)} are not ready."
+        if self.detail:
+            self.error_msg = self.error_msg + " " + self.detail
+        return self.error_msg

python/kserve/kserve/model_server.py

@@ -35,6 +35,7 @@
 from .protocol.model_repository_extension import ModelRepositoryExtension
 from .protocol.rest.server import UvicornServer
 from .utils import utils
+from kserve.errors import NoModelReady
 
 DEFAULT_HTTP_PORT = 8080
 DEFAULT_GRPC_PORT = 8081
@@ -226,13 +227,18 @@ def start(
             models: a list of models to register to the model server.
         """
         if isinstance(models, list):
+            at_least_one_model_ready = False
             for model in models:
                 if isinstance(model, BaseKServeModel):
-                    self.register_model(model)
-                    # pass whether to log request latency into the model
-                    model.enable_latency_logging = self.enable_latency_logging
+                    if model.ready:
+                        at_least_one_model_ready = True
+                        self.register_model(model)
+                        # pass whether to log request latency into the model
+                        model.enable_latency_logging = self.enable_latency_logging
                 else:
                     raise RuntimeError("Model type should be 'BaseKServeModel'")
+            if not at_least_one_model_ready and models:
+                raise NoModelReady(models)
         elif isinstance(models, dict):
             if all([isinstance(v, Deployment) for v in models.values()]):
                 # TODO: make this port number a variable

python/kserve/test/test_server.py

@@ -30,7 +30,7 @@
 from ray import serve
 
 from kserve import Model, ModelRepository, ModelServer
-from kserve.errors import InvalidInput
+from kserve.errors import InvalidInput, NoModelReady
 from kserve.model import PredictorProtocol
 from kserve.protocol.infer_type import (
     InferInput,
@@ -251,6 +251,13 @@ async def load(self, name: str) -> bool:
                 return False
 
 
+class DummyNeverReadyModel(Model):
+    def __init__(self, name):
+        super().__init__(name)
+        self.name = name
+        self.ready = False
+
+
 @pytest.mark.asyncio
 class TestModel:
     async def test_validate(self):
@@ -815,3 +822,12 @@ def test_explain(self, http_server_client):
             "/v1/models/TestModel:explain", content=b'{"instances":[[1,2]]}'
         )
         assert resp.status_code == 503
+
+
+class TestWithUnhealthModel:
+    def test_with_not_ready_model(self):
+        model = DummyNeverReadyModel("Dummy")
+        server = ModelServer()
+        with pytest.raises(NoModelReady) as exc_info:
+            server.start([model])
+        assert exc_info.type == NoModelReady

python/lgbserver/lgbserver/__main__.py

@@ -42,7 +42,7 @@
     try:
         model.load()
         # LightGBM doesn't support multi-process, so the number of http server workers should be 1.
-        kserve.ModelServer(workers=1).start([model] if model.ready else [])
+        kserve.ModelServer(workers=1).start([model])
     except ModelMissingError:
         logger.error(
             f"fail to load model {args.model_name} from dir {args.model_dir},"
@@ -51,4 +51,4 @@
         model_repository = LightGBMModelRepository(args.model_dir, args.nthread)
         # LightGBM doesn't support multi-process, so the number of http server workers should be 1.
         server = kserve.ModelServer(workers=1, registered_models=model_repository)
-        server.start([model] if model.ready else [])
+        server.start([model])

python/sklearnserver/sklearnserver/__main__.py

@@ -33,7 +33,7 @@
     model = SKLearnModel(args.model_name, args.model_dir)
     try:
         model.load()
-        kserve.ModelServer().start([model] if model.ready else [])
+        kserve.ModelServer().start([model])
 
     except ModelMissingError:
         logger.error(
@@ -43,4 +43,4 @@
 
         kserve.ModelServer(
             registered_models=SKLearnModelRepository(args.model_dir)
-        ).start([model] if model.ready else [])
+        ).start([model])

python/xgbserver/xgbserver/__main__.py

@@ -41,7 +41,7 @@
     model = XGBoostModel(args.model_name, args.model_dir, args.nthread)
     try:
         model.load()
-        kserve.ModelServer().start([model] if model.ready else [])
+        kserve.ModelServer().start([model])
     except ModelMissingError:
         logger.error(
             f"fail to locate model file for model {args.model_name} under dir {args.model_dir},"
@@ -50,4 +50,4 @@
 
         kserve.ModelServer(
             registered_models=XGBoostModelRepository(args.model_dir, args.nthread)
-        ).start([model] if model.ready else [])
+        ).start([model])