From 6c13169f3fc1aed421d2e3a7c16a422eb32c5800 Mon Sep 17 00:00:00 2001 From: Sivanantham Chinnaiyan Date: Sat, 10 Feb 2024 21:45:10 +0530 Subject: [PATCH] merge transformer, explainer, grpc and predictor tests Signed-off-by: Sivanantham Chinnaiyan --- .github/workflows/e2e-test.yml | 324 +++++++++++++-------------- test/e2e/helm/test_kserve_sklearn.py | 2 +- test/e2e/predictor/test_sklearn.py | 2 +- test/e2e/pytest.ini | 1 + test/e2e/qpext/test_qpext.py | 2 +- 5 files changed, 160 insertions(+), 171 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 703dd7e5449..366254d39ad 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -226,11 +226,31 @@ jobs: with: image: "kserve/paddleserver:${{ github.sha }}" + - name: Download custom model grpc image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/custom-model-grpc:${{ github.sha }}" + + - name: Download custom transformer grpc image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/custom-image-transformer-grpc:${{ github.sha }}" + - name: Download transformer image uses: ishworkh/docker-image-artifact-download@v1 with: image: "kserve/image-transformer:${{ github.sha }}" + - name: Download alibi image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/alibi-explainer:${{ github.sha }}" + + - name: Download art explainer image + uses: ishworkh/docker-image-artifact-download@v1 + with: + image: "kserve/art-explainer:${{ github.sha }}" + - name: Download success_200_isvc predictor image uses: ishworkh/docker-image-artifact-download@v1 with: @@ -254,7 +274,7 @@ jobs: - name: Run all E2E tests except graph timeout-minutes: 60 run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "fast or pmml or slow" "6" + ./test/scripts/gh-actions/run-e2e-tests.sh "fast or pmml or slow or explainer or transformer or mms or collocation or grpc" "6" - name: Run E2E tests for graph timeout-minutes: 120 @@ -346,7 +366,7 @@ jobs: run: | ./test/scripts/gh-actions/status-check.sh -# test-slow: +# test-explainer: # runs-on: ubuntu-latest # needs: # [kserve-image-build, predictor-runtime-build, explainer-runtime-build] @@ -364,117 +384,85 @@ jobs: # - uses: ./.github/actions/kserve-dep-setup # - uses: ./.github/actions/base-download # +# - name: Download sklearn server image +# uses: ishworkh/docker-image-artifact-download@v1 +# with: +# image: "kserve/sklearnserver:${{ github.sha }}" +# +# +# +# - name: Install Poetry and version plugin +# run: ./test/scripts/gh-actions/setup-poetry.sh +# +# - name: Install KServe +# run: | +# ./test/scripts/gh-actions/setup-kserve.sh +# +# kubectl get pods -n kserve +# kubectl describe pods -n kserve +# +# - name: Run E2E tests +# timeout-minutes: 40 +# run: | +# ./test/scripts/gh-actions/run-e2e-tests.sh "explainer" "6" # # - name: Check system status # if: always() # run: | # ./test/scripts/gh-actions/status-check.sh - test-explainer: - runs-on: ubuntu-latest - needs: - [kserve-image-build, predictor-runtime-build, explainer-runtime-build] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: "1.21" - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - uses: ./.github/actions/free-up-disk-space - - uses: ./.github/actions/minikube-setup - - uses: ./.github/actions/kserve-dep-setup - - uses: ./.github/actions/base-download - - - name: Download sklearn server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/sklearnserver:${{ github.sha }}" - - - name: Download alibi image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/alibi-explainer:${{ github.sha }}" - - - name: Download art explainer image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/art-explainer:${{ github.sha }}" - - - name: Install Poetry and version plugin - run: ./test/scripts/gh-actions/setup-poetry.sh - - - name: Install KServe - run: | - ./test/scripts/gh-actions/setup-kserve.sh - - kubectl get pods -n kserve - kubectl describe pods -n kserve - - - name: Run E2E tests - timeout-minutes: 40 - run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "explainer" "6" - - - name: Check system status - if: always() - run: | - ./test/scripts/gh-actions/status-check.sh - - test-transformer-mms: - runs-on: ubuntu-latest - needs: - [kserve-image-build, predictor-runtime-build, explainer-runtime-build] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: "1.21" - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - uses: ./.github/actions/free-up-disk-space - - uses: ./.github/actions/minikube-setup - - uses: ./.github/actions/kserve-dep-setup - - uses: ./.github/actions/base-download - - - name: Download sklearn server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/sklearnserver:${{ github.sha }}" - - - name: Download transformer image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/image-transformer:${{ github.sha }}" - - - name: Download xgb server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/xgbserver:${{ github.sha }}" - - - name: Install Poetry and version plugin - run: ./test/scripts/gh-actions/setup-poetry.sh - - - name: Install KServe - run: | - ./test/scripts/gh-actions/setup-kserve.sh - - kubectl get pods -n kserve - kubectl describe pods -n kserve - - - name: Run E2E tests - timeout-minutes: 40 - run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation" "6" - - - name: Check system status - if: always() - run: | - ./test/scripts/gh-actions/status-check.sh +# test-transformer-mms: +# runs-on: ubuntu-latest +# needs: +# [kserve-image-build, predictor-runtime-build, explainer-runtime-build] +# steps: +# - uses: actions/checkout@v3 +# - uses: actions/setup-go@v4 +# with: +# go-version: "1.21" +# - uses: actions/setup-python@v4 +# with: +# python-version: "3.9" +# +# - uses: ./.github/actions/free-up-disk-space +# - uses: ./.github/actions/minikube-setup +# - uses: ./.github/actions/kserve-dep-setup +# - uses: ./.github/actions/base-download +# +# - name: Download sklearn server image +# uses: ishworkh/docker-image-artifact-download@v1 +# with: +# image: "kserve/sklearnserver:${{ github.sha }}" +# +# - name: Download transformer image +# uses: ishworkh/docker-image-artifact-download@v1 +# with: +# image: "kserve/image-transformer:${{ github.sha }}" +# +# - name: Download xgb server image +# uses: ishworkh/docker-image-artifact-download@v1 +# with: +# image: "kserve/xgbserver:${{ github.sha }}" +# +# - name: Install Poetry and version plugin +# run: ./test/scripts/gh-actions/setup-poetry.sh +# +# - name: Install KServe +# run: | +# ./test/scripts/gh-actions/setup-kserve.sh +# +# kubectl get pods -n kserve +# kubectl describe pods -n kserve +# +# - name: Run E2E tests +# timeout-minutes: 40 +# run: | +# ./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation" "6" +# +# - name: Check system status +# if: always() +# run: | +# ./test/scripts/gh-actions/status-check.sh test-qpext: runs-on: ubuntu-latest @@ -520,67 +508,67 @@ jobs: run: | ./test/scripts/gh-actions/status-check.sh - test-grpc: - runs-on: ubuntu-latest - needs: - [kserve-image-build, predictor-runtime-build, explainer-runtime-build] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: "1.21" - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - uses: ./.github/actions/free-up-disk-space - - uses: ./.github/actions/minikube-setup - - uses: ./.github/actions/kserve-dep-setup - - uses: ./.github/actions/base-download - - - name: Download custom model grpc image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/custom-model-grpc:${{ github.sha }}" - - - name: Download custom transformer image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/image-transformer:${{ github.sha }}" - - - name: Download custom transformer grpc image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/custom-image-transformer-grpc:${{ github.sha }}" - - - name: Download xgb server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/xgbserver:${{ github.sha }}" - - - name: Download lgb server image - uses: ishworkh/docker-image-artifact-download@v1 - with: - image: "kserve/lgbserver:${{ github.sha }}" - - - name: Install Poetry and version plugin - run: ./test/scripts/gh-actions/setup-poetry.sh - - - name: Install KServe - run: | - ./test/scripts/gh-actions/setup-kserve.sh - kubectl get pods -n kserve - kubectl describe pods -n kserve - - - name: Run E2E tests - timeout-minutes: 40 - run: | - ./test/scripts/gh-actions/run-e2e-tests.sh "grpc" "6" - - - name: Check system status - if: always() - run: | - ./test/scripts/gh-actions/status-check.sh +# test-grpc: +# runs-on: ubuntu-latest +# needs: +# [kserve-image-build, predictor-runtime-build, explainer-runtime-build] +# steps: +# - uses: actions/checkout@v3 +# - uses: actions/setup-go@v4 +# with: +# go-version: "1.21" +# - uses: actions/setup-python@v4 +# with: +# python-version: "3.9" +# +# - uses: ./.github/actions/free-up-disk-space +# - uses: ./.github/actions/minikube-setup +# - uses: ./.github/actions/kserve-dep-setup +# - uses: ./.github/actions/base-download +# +# - name: Download custom model grpc image +# uses: ishworkh/docker-image-artifact-download@v1 +# with: +# image: "kserve/custom-model-grpc:${{ github.sha }}" +# +# - name: Download custom transformer image +# uses: ish workh/docker-image-artifact-download@v1 +# with: +# image: "kserve/image-transformer:${{ github.sha }}" +# +# - name: Download custom transformer grpc image +# uses: ishworkh/docker-image-artifact-download@v1 +# with: +# image: "kserve/custom-image-transformer-grpc:${{ github.sha }}" +# +# - name: Download xgb server image +# uses: ishworkh/docker-image-artifact-download@v1 +# with: +# image: "kserve/xgbserver:${{ github.sha }}" +# +# - name: Download lgb server image +# uses: ishworkh/docker-image-artifact-download@v1 +# with: +# image: "kserve/lgbserver:${{ github.sha }}" +# +# - name: Install Poetry and version plugin +# run: ./test/scripts/gh-actions/setup-poetry.sh +# +# - name: Install KServe +# run: | +# ./test/scripts/gh-actions/setup-kserve.sh +# kubectl get pods -n kserve +# kubectl describe pods -n kserve +# +# - name: Run E2E tests +# timeout-minutes: 40 +# run: | +# ./test/scripts/gh-actions/run-e2e-tests.sh "grpc" "6" +# +# - name: Check system status +# if: always() +# run: | +# ./test/scripts/gh-actions/status-check.sh test-with-helm: runs-on: ubuntu-latest diff --git a/test/e2e/helm/test_kserve_sklearn.py b/test/e2e/helm/test_kserve_sklearn.py index edbc9fbbaee..5b8d08d992a 100644 --- a/test/e2e/helm/test_kserve_sklearn.py +++ b/test/e2e/helm/test_kserve_sklearn.py @@ -33,7 +33,7 @@ @pytest.mark.helm def test_sklearn_kserve(): - service_name = "isvc-sklearn-kserve" + service_name = "isvc-sklearn-helm" predictor = V1beta1PredictorSpec( min_replicas=1, diff --git a/test/e2e/predictor/test_sklearn.py b/test/e2e/predictor/test_sklearn.py index 2fa6f0a57ed..94dd54ce399 100644 --- a/test/e2e/predictor/test_sklearn.py +++ b/test/e2e/predictor/test_sklearn.py @@ -61,7 +61,7 @@ def test_sklearn_kserve(): @pytest.mark.slow def test_sklearn_v2_mlserver(): - service_name = "isvc-sklearn-v2" + service_name = "sklearn-v2-mlserver" predictor = V1beta1PredictorSpec( min_replicas=1, sklearn=V1beta1SKLearnSpec( diff --git a/test/e2e/pytest.ini b/test/e2e/pytest.ini index cd845e389ff..fa4fd0043ab 100644 --- a/test/e2e/pytest.ini +++ b/test/e2e/pytest.ini @@ -11,3 +11,4 @@ markers = helm: helm e2e tests raw: raw e2e tests kourier: e2e tests using kourier as networking layer + collocation: transformer and predictor collocation e2e tests diff --git a/test/e2e/qpext/test_qpext.py b/test/e2e/qpext/test_qpext.py index ffd3834e5aa..18e5acff570 100644 --- a/test/e2e/qpext/test_qpext.py +++ b/test/e2e/qpext/test_qpext.py @@ -38,7 +38,7 @@ def test_qpext_kserve(): # test the qpext using the sklearn predictor - service_name = "isvc-sklearn-v2" + service_name = "sklearn-v2-metrics" predictor = V1beta1PredictorSpec( min_replicas=1, sklearn=V1beta1SKLearnSpec(