Fix mlserver fails due to model not ready

Signed-off-by: Sivanantham Chinnaiyan <[email protected]>

pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go

@@ -585,9 +585,6 @@ func (ir *IngressReconciler) Reconcile(isvc *v1beta1.InferenceService) (ctrl.Res
 			// When the ingress has already been marked Ready for this generation,
 			// then it must have been successfully probed. This exception necessary for the case
 			// of global resyncs.
-			// As this is an optimization, we don't worry about the ObservedGeneration
-			// skew we might see when the resource is actually in flux, we simply care
-			// about the steady state.
 		} else {
 			if isReady, err := probeIngress(isvc.Status.Address.URL.String()); err != nil {
 				return ctrl.Result{}, err

pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/suite_test.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2024 The KServe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package ingress
 
 import (

test/e2e/helm/test_kserve_sklearn.py

@@ -28,7 +28,7 @@
     constants
 )
 
-from ..common.utils import KSERVE_TEST_NAMESPACE, predict
+from ..common.utils import KSERVE_TEST_NAMESPACE, predict, get_cluster_ip
 
 
 @pytest.mark.helm
@@ -66,6 +66,8 @@ def test_sklearn_kserve():
     kserve_client.create(isvc)
     kserve_client.wait_isvc_ready(
         service_name, namespace=KSERVE_TEST_NAMESPACE)
+    kserve_client.wait_model_ready(service_name, model_name=service_name, isvc_namespace=KSERVE_TEST_NAMESPACE,
+                                   cluster_ip=get_cluster_ip(), protocol_version=protocol_version)
 
     res = predict(service_name, "./data/iris_input_v2.json",
                   protocol_version="v2")

test/e2e/predictor/test_lightgbm.py

@@ -14,6 +14,7 @@
 
 import json
 import os
+import time
 
 import numpy
 import pytest
@@ -139,6 +140,10 @@ def test_lightgbm_v2_runtime_mlserver():
     kserve_client.create(isvc)
     kserve_client.wait_isvc_ready(
         service_name, namespace=KSERVE_TEST_NAMESPACE)
+    # wait for model ready. Currently, wait_model_ready does not support path based routing
+    time.sleep(5)
+    # kserve_client.wait_model_ready(service_name, model_name=service_name, isvc_namespace=KSERVE_TEST_NAMESPACE,
+    #                                cluster_ip=get_cluster_ip(), protocol_version=protocol_version)
 
     res = predict(service_name, "./data/iris_input_v2.json",
                   protocol_version="v2")

test/e2e/predictor/test_mlflow.py

@@ -26,7 +26,7 @@
 from kubernetes.client import V1ResourceRequirements
 import pytest
 
-from ..common.utils import predict
+from ..common.utils import predict, get_cluster_ip
 from ..common.utils import KSERVE_TEST_NAMESPACE
 
 
@@ -62,6 +62,8 @@ def test_mlflow_v2_runtime_kserve():
     kserve_client = KServeClient(config_file=os.environ.get("KUBECONFIG", "~/.kube/config"))
     kserve_client.create(isvc)
     kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
+    kserve_client.wait_model_ready(service_name, model_name=service_name, isvc_namespace=KSERVE_TEST_NAMESPACE,
+                                   cluster_ip=get_cluster_ip(), protocol_version=protocol_version)
     res = predict(service_name, "./data/mlflow_input_v2.json", protocol_version=protocol_version)
     assert res["outputs"][0]["data"] == [5.576883936610762]
 

test/e2e/predictor/test_sklearn.py

@@ -25,7 +25,7 @@
 
 import kserve.protocol.grpc.grpc_predict_v2_pb2 as inference_pb2
 
-from ..common.utils import KSERVE_TEST_NAMESPACE, predict, predict_grpc
+from ..common.utils import KSERVE_TEST_NAMESPACE, predict, predict_grpc, get_cluster_ip
 
 
 @pytest.mark.predictor
@@ -88,8 +88,10 @@ def test_sklearn_v2_mlserver():
     kserve_client = KServeClient(config_file=os.environ.get("KUBECONFIG", "~/.kube/config"))
     kserve_client.create(isvc)
     kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
+    kserve_client.wait_model_ready(service_name, model_name=service_name, isvc_namespace=KSERVE_TEST_NAMESPACE,
+                                   cluster_ip=get_cluster_ip(), protocol_version=protocol_version)
 
-    res = predict(service_name, "./data/iris_input_v2.json", protocol_version="v2")
+    res = predict(service_name, "./data/iris_input_v2.json", protocol_version=protocol_version)
     assert res["outputs"][0]["data"] == [1, 1]
 
     kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
@@ -163,8 +165,10 @@ def test_sklearn_v2_runtime_mlserver():
     kserve_client = KServeClient(config_file=os.environ.get("KUBECONFIG", "~/.kube/config"))
     kserve_client.create(isvc)
     kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
+    kserve_client.wait_model_ready(service_name, model_name=service_name, isvc_namespace=KSERVE_TEST_NAMESPACE,
+                                   cluster_ip=get_cluster_ip(), protocol_version=protocol_version)
 
-    res = predict(service_name, "./data/iris_input_v2.json", protocol_version="v2")
+    res = predict(service_name, "./data/iris_input_v2.json", protocol_version=protocol_version)
     assert res["outputs"][0]["data"] == [1, 1]
 
     kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)

test/e2e/predictor/test_xgboost.py

@@ -14,6 +14,7 @@
 
 import json
 import os
+import time
 
 import pytest
 from kubernetes import client
@@ -93,9 +94,13 @@ def test_xgboost_v2_mlserver():
         config_file=os.environ.get("KUBECONFIG", "~/.kube/config"))
     kserve_client.create(isvc)
     kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
+    # wait for model ready. Currently, wait_model_ready does not support path based routing
+    time.sleep(5)
+    # kserve_client.wait_model_ready(service_name, model_name=service_name, isvc_namespace=KSERVE_TEST_NAMESPACE,
+    #                                cluster_ip=get_cluster_ip(), protocol_version=protocol_version)
 
     res = predict(service_name, "./data/iris_input_v2.json",
-                  protocol_version="v2")
+                  protocol_version=protocol_version)
     assert res["outputs"][0]["data"] == [1.0, 1.0]
 
     kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
@@ -173,9 +178,13 @@ def test_xgboost_v2_runtime_mlserver():
         config_file=os.environ.get("KUBECONFIG", "~/.kube/config"))
     kserve_client.create(isvc)
     kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
+    # wait for model ready. Currently, wait_model_ready does not support path based routing
+    time.sleep(5)
+    # kserve_client.wait_model_ready(service_name, model_name=service_name, isvc_namespace=KSERVE_TEST_NAMESPACE,
+    #                                cluster_ip=get_cluster_ip(), protocol_version=protocol_version)
 
     res = predict(service_name, "./data/iris_input_v2.json",
-                  protocol_version="v2")
+                  protocol_version=protocol_version)
     assert res["outputs"][0]["data"] == [1.0, 1.0]
 
     kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)