diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 6a34b46e5b3..9a1f951151f 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -172,7 +172,6 @@ jobs: [ kserve-image-build, predictor-runtime-build, - explainer-runtime-build, graph-tests-images-build, ] steps: @@ -226,10 +225,10 @@ jobs: with: image: "kserve/paddleserver:${{ github.sha }}" - - name: Download transformer image + - name: Download custom model grpc image uses: ishworkh/docker-image-artifact-download@v1 with: - image: "kserve/image-transformer:${{ github.sha }}" + image: "kserve/custom-model-grpc:${{ github.sha }}" - name: Download success_200_isvc predictor image uses: ishworkh/docker-image-artifact-download@v1 @@ -254,18 +253,86 @@ jobs: - name: Run all E2E tests except graph timeout-minutes: 60 run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "fast or pmml or slow" "4" + ./test/scripts/gh-actions/run-e2e-tests.sh "fast or pmml or slow or predictor" "6" - name: Run E2E tests for graph timeout-minutes: 120 run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "graph" "4" + ./test/scripts/gh-actions/run-e2e-tests.sh "graph" "6" - name: Check system status if: always() run: | ./test/scripts/gh-actions/status-check.sh + test-transformer-explainer-mms: + runs-on: ubuntu-latest + needs: + [kserve-image-build, predictor-runtime-build, explainer-runtime-build] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: "1.21" + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - uses: ./.github/actions/free-up-disk-space + - uses: ./.github/actions/minikube-setup + - uses: ./.github/actions/kserve-dep-setup + - uses: ./.github/actions/base-download + + - name: Download sklearn server image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/sklearnserver:${{ github.sha }}" + + - name: Download custom model grpc image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/custom-model-grpc:${{ github.sha }}" + + - name: Download transformer image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/image-transformer:${{ github.sha }}" + + - name: Download xgb server image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/xgbserver:${{ github.sha }}" + + - name: Download alibi image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/alibi-explainer:${{ github.sha }}" + + - name: Download art explainer image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/art-explainer:${{ github.sha }}" + + - name: Install Poetry and version plugin + run: ./test/scripts/gh-actions/setup-poetry.sh + + - name: Install KServe + run: | + ./test/scripts/gh-actions/setup-kserve.sh + + kubectl get pods -n kserve + kubectl describe pods -n kserve + + - name: Run E2E tests +# timeout-minutes: 40 + run: | + ./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation or explainer" "6" + + - name: Check system status + if: always() + run: | + ./test/scripts/gh-actions/status-check.sh + test-path-based-routing: runs-on: ubuntu-latest needs: @@ -339,137 +406,7 @@ jobs: - name: Run E2E tests with path-based routing timeout-minutes: 50 run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "fast" "4" - - - name: Check system status - if: always() - run: | - ./test/scripts/gh-actions/status-check.sh - -# test-slow: -# runs-on: ubuntu-latest -# needs: -# [kserve-image-build, predictor-runtime-build, explainer-runtime-build] -# steps: -# - uses: actions/checkout@v3 -# - uses: actions/setup-go@v4 -# with: -# go-version: "1.21" -# - uses: actions/setup-python@v4 -# with: -# python-version: "3.9" -# -# - uses: ./.github/actions/free-up-disk-space -# - uses: ./.github/actions/minikube-setup -# - uses: ./.github/actions/kserve-dep-setup -# - uses: ./.github/actions/base-download -# -# -# - name: Check system status -# if: always() -# run: | -# ./test/scripts/gh-actions/status-check.sh - - test-explainer: - runs-on: ubuntu-latest - needs: - [kserve-image-build, predictor-runtime-build, explainer-runtime-build] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: "1.21" - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - uses: ./.github/actions/free-up-disk-space - - uses: ./.github/actions/minikube-setup - - uses: ./.github/actions/kserve-dep-setup - - uses: ./.github/actions/base-download - - - name: Download sklearn server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/sklearnserver:${{ github.sha }}" - - - name: Download alibi image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/alibi-explainer:${{ github.sha }}" - - - name: Download art explainer image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/art-explainer:${{ github.sha }}" - - - name: Install Poetry and version plugin - run: ./test/scripts/gh-actions/setup-poetry.sh - - - name: Install KServe - run: | - ./test/scripts/gh-actions/setup-kserve.sh - - kubectl get pods -n kserve - kubectl describe pods -n kserve - - - name: Run E2E tests - timeout-minutes: 40 - run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "explainer" "4" - - - name: Check system status - if: always() - run: | - ./test/scripts/gh-actions/status-check.sh - - test-transformer-mms: - runs-on: ubuntu-latest - needs: - [kserve-image-build, predictor-runtime-build, explainer-runtime-build] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: "1.21" - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - uses: ./.github/actions/free-up-disk-space - - uses: ./.github/actions/minikube-setup - - uses: ./.github/actions/kserve-dep-setup - - uses: ./.github/actions/base-download - - - name: Download sklearn server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/sklearnserver:${{ github.sha }}" - - - name: Download transformer image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/image-transformer:${{ github.sha }}" - - - name: Download xgb server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/xgbserver:${{ github.sha }}" - - - name: Install Poetry and version plugin - run: ./test/scripts/gh-actions/setup-poetry.sh - - - name: Install KServe - run: | - ./test/scripts/gh-actions/setup-kserve.sh - - kubectl get pods -n kserve - kubectl describe pods -n kserve - - - name: Run E2E tests - timeout-minutes: 40 - run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation" "4" + ./test/scripts/gh-actions/run-e2e-tests.sh "fast" "6" - name: Check system status if: always() @@ -520,68 +457,6 @@ jobs: run: | ./test/scripts/gh-actions/status-check.sh - test-grpc: - runs-on: ubuntu-latest - needs: - [kserve-image-build, predictor-runtime-build, explainer-runtime-build] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: "1.21" - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - uses: ./.github/actions/free-up-disk-space - - uses: ./.github/actions/minikube-setup - - uses: ./.github/actions/kserve-dep-setup - - uses: ./.github/actions/base-download - - - name: Download custom model grpc image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/custom-model-grpc:${{ github.sha }}" - - - name: Download custom transformer image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/image-transformer:${{ github.sha }}" - - - name: Download custom transformer grpc image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/custom-image-transformer-grpc:${{ github.sha }}" - - - name: Download xgb server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/xgbserver:${{ github.sha }}" - - - name: Download lgb server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/lgbserver:${{ github.sha }}" - - - name: Install Poetry and version plugin - run: ./test/scripts/gh-actions/setup-poetry.sh - - - name: Install KServe - run: | - ./test/scripts/gh-actions/setup-kserve.sh - kubectl get pods -n kserve - kubectl describe pods -n kserve - - - name: Run E2E tests - timeout-minutes: 40 - run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "grpc" "4" - - - name: Check system status - if: always() - run: | - ./test/scripts/gh-actions/status-check.sh - test-with-helm: runs-on: ubuntu-latest needs: @@ -649,6 +524,11 @@ jobs: with: image: "kserve/image-transformer:${{ github.sha }}" + - name: Download custom model grpc image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/custom-model-grpc:${{ github.sha }}" + - name: Install Poetry and version plugin run: ./test/scripts/gh-actions/setup-poetry.sh @@ -662,7 +542,7 @@ jobs: - name: Run E2E tests timeout-minutes: 40 run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "raw" "4" + ./test/scripts/gh-actions/run-e2e-tests.sh "raw" "6" - name: Check system status if: always() @@ -731,7 +611,7 @@ jobs: --output=jsonpath="{.items[0].status.podIP}"):$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \ --output=jsonpath="{.items[0].spec.containers[0].ports[0].containerPort}") - ./test/scripts/gh-actions/run-e2e-tests.sh "kourier" "4" + ./test/scripts/gh-actions/run-e2e-tests.sh "kourier" "6" kubectl get pods -n kserve diff --git a/test/e2e/custom/test_custom_model_grpc.py b/test/e2e/custom/test_custom_model_grpc.py index 7be5bb663ab..89a8505cd31 100644 --- a/test/e2e/custom/test_custom_model_grpc.py +++ b/test/e2e/custom/test_custom_model_grpc.py @@ -31,6 +31,7 @@ @pytest.mark.grpc +@pytest.mark.predictor def test_custom_model_grpc(): service_name = "custom-model-grpc" model_name = "custom-model" @@ -92,6 +93,7 @@ def test_custom_model_grpc(): @pytest.mark.grpc +@pytest.mark.transformer def test_predictor_grpc_with_transformer_grpc(): service_name = "model-grpc-trans-grpc" model_name = "custom-model" @@ -174,6 +176,7 @@ def test_predictor_grpc_with_transformer_grpc(): @pytest.mark.grpc +@pytest.mark.transformer def test_predictor_grpc_with_transformer_http(): service_name = "model-grpc-trans-http" model_name = "custom-model" diff --git a/test/e2e/helm/test_kserve_sklearn.py b/test/e2e/helm/test_kserve_sklearn.py index edbc9fbbaee..5b8d08d992a 100644 --- a/test/e2e/helm/test_kserve_sklearn.py +++ b/test/e2e/helm/test_kserve_sklearn.py @@ -33,7 +33,7 @@ @pytest.mark.helm def test_sklearn_kserve(): - service_name = "isvc-sklearn-kserve" + service_name = "isvc-sklearn-helm" predictor = V1beta1PredictorSpec( min_replicas=1, diff --git a/test/e2e/logger/test_logger.py b/test/e2e/logger/test_logger.py index 8457bdd2954..4d08256ff6c 100644 --- a/test/e2e/logger/test_logger.py +++ b/test/e2e/logger/test_logger.py @@ -56,7 +56,7 @@ def test_kserve_logger(): min_replicas=1, logger=V1beta1LoggerSpec( mode="all", - url="http://message-dumper."+KSERVE_TEST_NAMESPACE+".svc.cluster.local" + url=f"http://{msg_dumper}."+KSERVE_TEST_NAMESPACE+".svc.cluster.local" ), sklearn=V1beta1SKLearnSpec( storage_uri='gs://kfserving-examples/models/sklearn/1.0/model', diff --git a/test/e2e/predictor/test_grpc.py b/test/e2e/predictor/test_grpc.py index 58a31cbd64a..7165f184751 100644 --- a/test/e2e/predictor/test_grpc.py +++ b/test/e2e/predictor/test_grpc.py @@ -33,11 +33,12 @@ @pytest.mark.grpc -def test_custom_model_grpc(): +@pytest.mark.predictor +def test_custom_model_grpc_logger(): service_name = "custom-grpc-logger" model_name = "custom-model" - msg_dumper = 'message-dumper' + msg_dumper = 'message-dumper-grpc' logger_predictor = V1beta1PredictorSpec( min_replicas=1, containers=[V1Container(name="kserve-container", @@ -60,7 +61,7 @@ def test_custom_model_grpc(): predictor = V1beta1PredictorSpec( logger=V1beta1LoggerSpec( mode="all", - url="http://message-dumper." + KSERVE_TEST_NAMESPACE + ".svc.cluster.local" + url=f"http://{msg_dumper}." + KSERVE_TEST_NAMESPACE + ".svc.cluster.local" ), containers=[ V1Container( diff --git a/test/e2e/predictor/test_lightgbm.py b/test/e2e/predictor/test_lightgbm.py index fdabef88a21..0396a9827e7 100644 --- a/test/e2e/predictor/test_lightgbm.py +++ b/test/e2e/predictor/test_lightgbm.py @@ -197,6 +197,7 @@ def test_lightgbm_v2_kserve(): @pytest.mark.grpc +@pytest.mark.predictor def test_lightgbm_v2_grpc(): service_name = "isvc-lightgbm-v2-grpc" model_name = "lightgbm" diff --git a/test/e2e/predictor/test_raw_deployment.py b/test/e2e/predictor/test_raw_deployment.py index fe4754364a5..fe9879b9278 100644 --- a/test/e2e/predictor/test_raw_deployment.py +++ b/test/e2e/predictor/test_raw_deployment.py @@ -112,7 +112,8 @@ def test_raw_deployment_runtime_kserve(): @pytest.mark.grpc -def test_isvc_with_multiple_container_port(): +@pytest.mark.raw +def test_raw_isvc_with_multiple_container_port(): service_name = "raw-multiport-custom-model" model_name = "custom-model" diff --git a/test/e2e/predictor/test_sklearn.py b/test/e2e/predictor/test_sklearn.py index 2fa6f0a57ed..94dd54ce399 100644 --- a/test/e2e/predictor/test_sklearn.py +++ b/test/e2e/predictor/test_sklearn.py @@ -61,7 +61,7 @@ def test_sklearn_kserve(): @pytest.mark.slow def test_sklearn_v2_mlserver(): - service_name = "isvc-sklearn-v2" + service_name = "sklearn-v2-mlserver" predictor = V1beta1PredictorSpec( min_replicas=1, sklearn=V1beta1SKLearnSpec( diff --git a/test/e2e/predictor/test_torchserve.py b/test/e2e/predictor/test_torchserve.py index dbdd5806ed4..7e3c113752e 100644 --- a/test/e2e/predictor/test_torchserve.py +++ b/test/e2e/predictor/test_torchserve.py @@ -107,6 +107,7 @@ def test_torchserve_v2_kserve(): @pytest.mark.grpc +@pytest.mark.predictor def test_torchserve_grpc_v2(): service_name = "mnist-grpc" model_name = "mnist" diff --git a/test/e2e/predictor/test_xgboost.py b/test/e2e/predictor/test_xgboost.py index 35f229a7927..1f92ef85beb 100644 --- a/test/e2e/predictor/test_xgboost.py +++ b/test/e2e/predictor/test_xgboost.py @@ -216,6 +216,7 @@ def test_xgboost_v2(): @pytest.mark.grpc +@pytest.mark.predictor def test_xgboost_v2_grpc(): service_name = "isvc-xgboost-v2-grpc" model_name = "xgboost" diff --git a/test/e2e/pytest.ini b/test/e2e/pytest.ini index cd845e389ff..fa4fd0043ab 100644 --- a/test/e2e/pytest.ini +++ b/test/e2e/pytest.ini @@ -11,3 +11,4 @@ markers = helm: helm e2e tests raw: raw e2e tests kourier: e2e tests using kourier as networking layer + collocation: transformer and predictor collocation e2e tests diff --git a/test/e2e/qpext/test_qpext.py b/test/e2e/qpext/test_qpext.py index ffd3834e5aa..18e5acff570 100644 --- a/test/e2e/qpext/test_qpext.py +++ b/test/e2e/qpext/test_qpext.py @@ -38,7 +38,7 @@ def test_qpext_kserve(): # test the qpext using the sklearn predictor - service_name = "isvc-sklearn-v2" + service_name = "sklearn-v2-metrics" predictor = V1beta1PredictorSpec( min_replicas=1, sklearn=V1beta1SKLearnSpec(