diff --git a/.github/actions/kserve-dep-setup/action.yml b/.github/actions/kserve-dep-setup/action.yml
index c3cf92a4f18..37a08dd830f 100644
--- a/.github/actions/kserve-dep-setup/action.yml
+++ b/.github/actions/kserve-dep-setup/action.yml
@@ -5,6 +5,10 @@ inputs:
     description: 'Network layer to install'
     required: true
     default: 'istio'
+  deployment-mode:
+    description:  'Kserve deployment mode. Supported values are serverless and raw'
+    required: false
+    default: 'serverless'
 runs:
   using: "composite"
   steps:
@@ -15,7 +19,7 @@ runs:
 
         if [[ "${{ inputs.network-layer }}" == "istio" ]]; then
             echo "Selected network layer ${{ inputs.network-layer }}"
-            ./test/scripts/gh-actions/setup-deps.sh
+            ./test/scripts/gh-actions/setup-deps.sh ${{ inputs.deployment-mode }}
         
             kubectl get pods -n istio-system
             kubectl describe svc -n istio-system istio-ingressgateway
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index 02a5301b736..7306a2ff7c8 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -275,6 +275,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
+      - name: Free-up disk space
+        uses: ./.github/actions/free-up-disk-space
+
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
@@ -285,18 +288,15 @@ jobs:
         with:
           python-version: "3.9"
 
-      - name: Free-up disk space
-        uses: ./.github/actions/free-up-disk-space
-
       - name: Setup Minikube
         uses: ./.github/actions/minikube-setup
 
-      - name: Download base images
-        uses: ./.github/actions/base-download
-
       - name: KServe dependency setup
         uses: ./.github/actions/kserve-dep-setup
 
+      - name: Download base images
+        uses: ./.github/actions/base-download
+
       - name: Download predictor artifacts
         uses: actions/download-artifact@v4
         with:
@@ -670,6 +670,8 @@ jobs:
       - uses: ./.github/actions/free-up-disk-space
       - uses: ./.github/actions/minikube-setup
       - uses: ./.github/actions/kserve-dep-setup
+        with:
+          deployment-mode: "raw"
       - uses: ./.github/actions/base-download
 
       - name: Download sklearn server image
diff --git a/test/e2e/common/utils.py b/test/e2e/common/utils.py
index cfa990b83e2..2870de3f46c 100644
--- a/test/e2e/common/utils.py
+++ b/test/e2e/common/utils.py
@@ -13,7 +13,6 @@
 import json
 import logging
 import os
-import time
 from urllib.parse import urlparse
 
 import grpc
@@ -81,8 +80,6 @@ def predict_str(service_name, input_json, protocol_version="v1",
         namespace=KSERVE_TEST_NAMESPACE,
         version=version,
     )
-    # temporary sleep until this is fixed https://github.com/kserve/kserve/issues/604
-    time.sleep(10)
     cluster_ip, host, path = get_isvc_endpoint(isvc)
     headers = {"Host": host, "Content-Type": "application/json"}
 
@@ -151,8 +148,6 @@ def explain_response(service_name, input_json):
         namespace=KSERVE_TEST_NAMESPACE,
         version=constants.KSERVE_V1BETA1_VERSION,
     )
-    # temporary sleep until this is fixed https://github.com/kserve/kserve/issues/604
-    time.sleep(10)
     cluster_ip, host, _ = get_isvc_endpoint(isvc)
     url = "http://{}/v1/models/{}:explain".format(cluster_ip, service_name)
     headers = {"Host": host}
diff --git a/test/scripts/gh-actions/setup-deps.sh b/test/scripts/gh-actions/setup-deps.sh
index aa2f4b69fa8..a5ecd6b14d6 100755
--- a/test/scripts/gh-actions/setup-deps.sh
+++ b/test/scripts/gh-actions/setup-deps.sh
@@ -22,6 +22,7 @@ set -o nounset
 set -o pipefail
 
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
+DEPLOYMENT_MODE="$1"
 
 ISTIO_VERSION="1.19.4"
 CERT_MANAGER_VERSION="v1.5.0"
@@ -56,18 +57,23 @@ spec:
   controller: istio.io/ingress-controller
 EOF
 
-source  ./test/scripts/gh-actions/install-knative-operator.sh
-
-echo "Installing Knative serving ..."
-kubectl apply -f ./test/overlays/knative/knative-serving-istio.yaml
-# sleep to avoid running kubectl wait before pods are created
-sleep 15
-
-echo "Waiting for Knative to be ready ..."
-kubectl wait --for=condition=Ready pods --all --timeout=400s -n knative-serving -l 'app in (webhook, activator,autoscaler,autoscaler-hpa,controller,net-istio-controller,net-istio-webhook)'
-
-# echo "Add knative hpa..."
-# kubectl apply -f https://github.com/knative/serving/releases/download/knative-v1.0.0/serving-hpa.yaml
+shopt -s nocasematch
+if [[ $DEPLOYMENT_MODE == "serverless" ]];then
+  source  ./test/scripts/gh-actions/install-knative-operator.sh
+
+  echo "Installing Knative serving ..."
+  kubectl apply -f ./test/overlays/knative/knative-serving-istio.yaml
+  # sleep to avoid running kubectl wait before pods are created
+  sleep 15
+  echo "Waiting for Knative to be ready ..."
+  kubectl wait --for=condition=Ready pods --all --timeout=400s -n knative-serving -l 'app in (webhook, activator,autoscaler,autoscaler-hpa,controller,net-istio-controller,net-istio-webhook)'
+  # echo "Add knative hpa..."
+  # kubectl apply -f https://github.com/knative/serving/releases/download/knative-v1.0.0/serving-hpa.yaml
+fi
+if [[ $DEPLOYMENT_MODE == "raw" ]];then
+  kubectl patch cm -n kserve inferenceservice-config --patch='{"data": {"deploy": "{\"defaultDeploymentMode\": \"RawDeployment\"}"}}'
+fi
+shopt -u nocasematch
 
 echo "Installing cert-manager ..."
 kubectl create namespace cert-manager