e2e-test.yml

name: E2E Tests

on:
  pull_request:
    branches: [master, release*]
    paths:
      - "**"
#      - "!.github/**"
      - "!docs/**"
      - "!**.md"
      - ".github/workflows/e2e-test.yml"
  workflow_dispatch:

env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  kserve-image-build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Build KServe images
        run: |
          ./test/scripts/gh-actions/build-images.sh
          docker image ls
          cat ./config/overlays/test/configmap/inferenceservice.yaml

      - name: Upload controller image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/kserve-controller:${{ github.sha }}"

      - name: Upload agent image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/agent:${{ github.sha }}"

      - name: Upload storage initializer image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/storage-initializer:${{ github.sha }}"

      - name: Upload router image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/router:${{ github.sha }}"

  poetry-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Check poetry lock file consistency
        run: ./test/scripts/gh-actions/check-poetry-lockfile.sh

  predictor-runtime-build:
    runs-on: ubuntu-latest
    needs: [poetry-check]
    steps:
      - uses: actions/checkout@v3

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Build runtime server images
        run: |
          ./test/scripts/gh-actions/build-server-runtimes.sh predictor,transformer
          docker image ls
          cat ./config/overlays/test/configmap/inferenceservice.yaml

      - name: Upload sklearn server image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/sklearnserver:${{ github.sha }}"

      - name: Upload xgb server image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/xgbserver:${{ github.sha }}"

      - name: Upload lgb server image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/lgbserver:${{ github.sha }}"

      - name: Upload pmml server image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/pmmlserver:${{ github.sha }}"

      - name: Upload paddle image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/paddleserver:${{ github.sha }}"

      - name: Upload image transformer image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/image-transformer:${{ github.sha }}"

      - name: Upload custom model grpc image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/custom-model-grpc:${{ github.sha }}"

      - name: Upload custom model transformer grpc image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/custom-image-transformer-grpc:${{ github.sha }}"

  explainer-runtime-build:
    runs-on: ubuntu-latest
    needs: [poetry-check]
    steps:
      - uses: actions/checkout@v3

      - name: Build runtime server images
        run: |
          ./test/scripts/gh-actions/build-server-runtimes.sh explainer
          docker image ls
          cat ./config/overlays/test/configmap/inferenceservice.yaml

      - name: Upload alibi image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/alibi-explainer:${{ github.sha }}"

      - name: Upload art explainer image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/art-explainer:${{ github.sha }}"

  graph-tests-images-build:
    runs-on: ubuntu-latest
    needs: [poetry-check]
    steps:
      - uses: actions/checkout@v3

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Build test images needed for graph tests
        run: |
          ./test/scripts/gh-actions/build-graph-tests-images.sh
          docker image ls

      - name: Upload success_200_isvc predictor image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/success-200-isvc:${{ github.sha }}"

      - name: Upload error_404_isvc predictor image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "kserve/error-404-isvc:${{ github.sha }}"

  test-fast:
    runs-on: ubuntu-latest
    needs:
      [
        kserve-image-build,
        predictor-runtime-build,
        explainer-runtime-build,
        graph-tests-images-build,
      ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Setup Go
        uses: actions/setup-go@v4
        with:
          go-version: "1.21"

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download sklearn server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/sklearnserver:${{ github.sha }}"

      - name: Download xgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/xgbserver:${{ github.sha }}"

      - name: Download lgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/lgbserver:${{ github.sha }}"

      - name: Download pmml image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/pmmlserver:${{ github.sha }}"

      - name: Download transformer image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/image-transformer:${{ github.sha }}"

      - name: Download success_200_isvc predictor image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/success-200-isvc:${{ github.sha }}"

      - name: Download error_404_isvc predictor image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/error-404-isvc:${{ github.sha }}"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run all E2E tests except graph
        timeout-minutes: 60
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "fast or pmml" "2"

      - name: Run E2E tests for graph
        timeout-minutes: 120
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "graph" "2"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-path-based-routing:
    runs-on: ubuntu-latest
    needs:
      [
        kserve-image-build,
        predictor-runtime-build,
        explainer-runtime-build,
        graph-tests-images-build,
      ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Setup Go
        uses: actions/setup-go@v4
        with:
          go-version: "1.21"

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download sklearn server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/sklearnserver:${{ github.sha }}"

      - name: Download xgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/xgbserver:${{ github.sha }}"

      - name: Download lgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/lgbserver:${{ github.sha }}"

      - name: Download transformer image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/image-transformer:${{ github.sha }}"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Patch inferenceservice config
        run: |
          kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-ingress.yaml
          kubectl describe configmaps -n kserve inferenceservice-config

      - name: Run E2E tests with path-based routing
        timeout-minutes: 50
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "fast" "2"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-slow:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: "1.21"
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - uses: ./.github/actions/free-up-disk-space
      - uses: ./.github/actions/minikube-setup
      - uses: ./.github/actions/kserve-dep-setup
      - uses: ./.github/actions/base-download

      - name: Download sklearn server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/sklearnserver:${{ github.sha }}"

      - name: Download xgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/xgbserver:${{ github.sha }}"

      - name: Download lgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/lgbserver:${{ github.sha }}"

      - name: Download paddle image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/paddleserver:${{ github.sha }}"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 90
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "slow" "2"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-explainer:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: "1.21"
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - uses: ./.github/actions/free-up-disk-space
      - uses: ./.github/actions/minikube-setup
      - uses: ./.github/actions/kserve-dep-setup
      - uses: ./.github/actions/base-download

      - name: Download sklearn server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/sklearnserver:${{ github.sha }}"

      - name: Download alibi image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/alibi-explainer:${{ github.sha }}"

      - name: Download art explainer image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/art-explainer:${{ github.sha }}"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 40
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "explainer" "2"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-transformer-mms:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: "1.21"
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - uses: ./.github/actions/free-up-disk-space
      - uses: ./.github/actions/minikube-setup
      - uses: ./.github/actions/kserve-dep-setup
      - uses: ./.github/actions/base-download

      - name: Download sklearn server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/sklearnserver:${{ github.sha }}"

      - name: Download transformer image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/image-transformer:${{ github.sha }}"

      - name: Download xgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/xgbserver:${{ github.sha }}"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 40
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation" "2"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-qpext:
    runs-on: ubuntu-latest
    needs: [kserve-image-build, predictor-runtime-build]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: "1.21"
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      - uses: ./.github/actions/free-up-disk-space
      - uses: ./.github/actions/minikube-setup
      - uses: ./.github/actions/kserve-dep-setup
      - uses: ./.github/actions/base-download
      - name: Build queue proxy extension image
        run: |
          ./test/scripts/gh-actions/build-qpext-image.sh
          docker image ls
      - name: Download sklearn server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/sklearnserver:${{ github.sha }}"
      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh
      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh
          kubectl get pods -n kserve
          kubectl describe pods -n kserve
      - name: Patch qpext image
        run: |
          kubectl patch configmaps -n knative-serving config-deployment --patch '{"data": {"queue-sidecar-image": "kserve/qpext:${{ github.sha }}"}}'
          kubectl describe configmaps -n knative-serving config-deployment
      - name: Run queue proxy extension E2E tests
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-qpext-test.sh

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-grpc:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: "1.21"
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - uses: ./.github/actions/free-up-disk-space
      - uses: ./.github/actions/minikube-setup
      - uses: ./.github/actions/kserve-dep-setup
      - uses: ./.github/actions/base-download

      - name: Download custom model grpc image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/custom-model-grpc:${{ github.sha }}"

      - name: Download custom transformer image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/image-transformer:${{ github.sha }}"

      - name: Download custom transformer grpc image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/custom-image-transformer-grpc:${{ github.sha }}"

      - name: Download xgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/xgbserver:${{ github.sha }}"

      - name: Download lgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/lgbserver:${{ github.sha }}"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh
          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 40
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "grpc" "2"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-with-helm:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: "1.21"
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - uses: ./.github/actions/free-up-disk-space
      - uses: ./.github/actions/minikube-setup
      - uses: ./.github/actions/kserve-dep-setup
      - uses: ./.github/actions/base-download

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install Kserve from helm
        run: |
          ./test/scripts/gh-actions/setup-modelmesh-dep.sh
          ./test/scripts/gh-actions/setup-kserve-helm.sh
          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 50
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "helm"
          kubectl get pods -n kserve

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-raw:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: "1.21"
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - uses: ./.github/actions/free-up-disk-space
      - uses: ./.github/actions/minikube-setup
      - uses: ./.github/actions/kserve-dep-setup
      - uses: ./.github/actions/base-download

      - name: Download sklearn server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/sklearnserver:${{ github.sha }}"

      - name: Download transformer image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/image-transformer:${{ github.sha }}"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 40
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "raw" "2"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-kourier:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, graph-tests-images-build]
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-go@v2
        with:
          go-version: "1.21"
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - uses: ./.github/actions/free-up-disk-space
      - uses: ./.github/actions/minikube-setup
      - uses: ./.github/actions/kserve-dep-setup
        with:
          network-layer: "kourier"
      - uses: ./.github/actions/base-download

      - name: Download sklearn server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/sklearnserver:${{ github.sha }}"

      - name: Download xgb server image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/xgbserver:${{ github.sha }}"

      - name: Download success_200_isvc predictor image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/success-200-isvc:${{ github.sha }}"

      - name: Download error_404_isvc predictor image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "kserve/error-404-isvc:${{ github.sha }}"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Patch inferenceservice config to disable istio
        run: |
          kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-disable-istio.yaml
          kubectl describe configmaps -n kserve inferenceservice-config

      - name: Run E2E tests
        timeout-minutes: 120
        run: |
          export KSERVE_INGRESS_HOST_PORT=$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \
          --output=jsonpath="{.items[0].status.podIP}"):$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \
          --output=jsonpath="{.items[0].spec.containers[0].ports[0].containerPort}")

          ./test/scripts/gh-actions/run-e2e-tests.sh "kourier" "2"

          kubectl get pods -n kserve

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh "kourier"