e2e-test.yml

name: E2E Tests

on:
  pull_request:
    branches: [master, release*]
    paths:
      - "**"
      - "!.github/**"
      - "!docs/**"
      - "!**.md"
      - ".github/workflows/e2e-test.yml"
  workflow_dispatch:

env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  DOCKER_IMAGES_PATH: /mnt/docker-images
  DOCKER_REPO: kserve
  # artifact prefixes for bulk download
  PREDICTOR_ARTIFACT_PREFIX: pred
  EXPLAINER_ARTIFACT_PREFIX: exp
  TRANSFORMER_ARTIFACT_PREFIX: trans
  GRAPH_ARTIFACT_PREFIX: graph
  BASE_ARTIFACT_PREFIX: base
  # Controller images
  CONTROLLER_IMG: kserve-controller
  STORAGE_INIT_IMG: storage-initializer
  AGENT_IMG: agent
  ROUTER_IMG: router
  # Predictor runtime server images
  SKLEARN_IMG: sklearnserver
  XGB_IMG: xgbserver
  LGB_IMG: lgbserver
  PMML_IMG: pmmlserver
  PADDLE_IMG: paddleserver
  CUSTOM_MODEL_GRPC_IMG: custom-model-grpc
  # Explainer images
  ALIBI_IMG: alibi-explainer
  ART_IMG: art-explainer
  # Transformer images
  IMAGE_TRANSFORMER_IMG: image-transformer
  CUSTOM_TRANSFORMER_GRPC_IMG: custom-image-transformer-grpc
  # Graph images
  SUCCESS_200_ISVC_IMG: success-200-isvc
  ERROR_404_ISVC_IMG: error-404-isvc

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  kserve-image-build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build KServe images
        run: |
          sudo mkdir -p ${DOCKER_IMAGES_PATH}
          sudo chown -R $USER ${DOCKER_IMAGES_PATH}
          ./test/scripts/gh-actions/build-images.sh
          docker image ls
          sudo ls -lh ${DOCKER_IMAGES_PATH}

      - name: Upload controller image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.BASE_ARTIFACT_PREFIX }}-${{ env.CONTROLLER_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.CONTROLLER_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload agent image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.BASE_ARTIFACT_PREFIX }}-${{ env.AGENT_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.AGENT_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload storage initializer image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.BASE_ARTIFACT_PREFIX }}-${{ env.STORAGE_INIT_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.STORAGE_INIT_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload router image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.BASE_ARTIFACT_PREFIX }}-${{ env.ROUTER_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.ROUTER_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

  poetry-check:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Check poetry lock file consistency
        run: ./test/scripts/gh-actions/check-poetry-lockfile.sh

  predictor-runtime-build:
    runs-on: ubuntu-latest
    needs: [poetry-check]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build runtime server images
        run: |
          sudo mkdir -p ${DOCKER_IMAGES_PATH}
          sudo chown -R $USER ${DOCKER_IMAGES_PATH}
          ./test/scripts/gh-actions/build-server-runtimes.sh predictor,transformer
          docker image ls
          sudo ls -lh ${DOCKER_IMAGES_PATH}

      - name: Upload sklearn artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.SKLEARN_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload xgb server image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.XGB_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload lgb server image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.LGB_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.LGB_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload pmml server image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.PMML_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.PMML_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload paddle image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.PADDLE_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.PADDLE_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload image transformer image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.TRANSFORMER_ARTIFACT_PREFIX }}-${{ env.IMAGE_TRANSFORMER_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.IMAGE_TRANSFORMER_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload custom model grpc image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.CUSTOM_MODEL_GRPC_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.CUSTOM_MODEL_GRPC_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload custom model transformer grpc image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.TRANSFORMER_ARTIFACT_PREFIX }}-${{ env.CUSTOM_TRANSFORMER_GRPC_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.CUSTOM_TRANSFORMER_GRPC_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

  explainer-runtime-build:
    runs-on: ubuntu-latest
    needs: [poetry-check]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build runtime server images
        run: |
          sudo mkdir -p ${DOCKER_IMAGES_PATH}
          sudo chown -R $USER ${DOCKER_IMAGES_PATH}
          ./test/scripts/gh-actions/build-server-runtimes.sh explainer
          docker image ls
          sudo ls -lh ${DOCKER_IMAGES_PATH}

      - name: Upload alibi image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.EXPLAINER_ARTIFACT_PREFIX }}-${{ env.ALIBI_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.ALIBI_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload art explainer image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.EXPLAINER_ARTIFACT_PREFIX }}-${{ env.ART_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.ART_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

  graph-tests-images-build:
    runs-on: ubuntu-latest
    needs: [poetry-check]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build test images needed for graph tests
        run: |
          sudo mkdir -p ${DOCKER_IMAGES_PATH}
          sudo chown -R $USER ${DOCKER_IMAGES_PATH}
          ./test/scripts/gh-actions/build-graph-tests-images.sh
          docker image ls
          sudo ls -lh ${DOCKER_IMAGES_PATH}

      - name: Upload success_200_isvc predictor image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.GRAPH_ARTIFACT_PREFIX }}-${{ env.SUCCESS_200_ISVC_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.SUCCESS_200_ISVC_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload error_404_isvc predictor image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.GRAPH_ARTIFACT_PREFIX }}-${{ env.ERROR_404_ISVC_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.ERROR_404_ISVC_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

  test-predictor:
    runs-on: ubuntu-latest
    needs:
      [
        kserve-image-build,
        predictor-runtime-build,
      ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Download predictor artifacts
        uses: actions/download-artifact@v4
        with:
          path: ./tmp
          pattern: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-*
          merge-multiple: true

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run predictor E2E tests
        timeout-minutes: 40
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "predictor" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-transformer-explainer-mms:
      runs-on: ubuntu-latest
      needs:
        [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
      steps:
        - name: Checkout source
          uses: actions/checkout@v4

        - name: Free-up disk space
          uses: ./.github/actions/free-up-disk-space

        - name: Setup Go
          uses: actions/setup-go@v5
          with:
            go-version: "1.21"
        - name: Setup Python
          uses: actions/setup-python@v5
          with:
            python-version: "3.9"

        - name: Setup Minikube
          uses: ./.github/actions/minikube-setup

        - name: KServe dependency setup
          uses: ./.github/actions/kserve-dep-setup

        - name: Download base images
          uses: ./.github/actions/base-download

        - name: Download transformer and explainer artifacts
          uses: actions/download-artifact@v4
          with:
            path: ./tmp
            pattern: +(${{ env.TRANSFORMER_ARTIFACT_PREFIX }}|${{ env.EXPLAINER_ARTIFACT_PREFIX }})-*
            merge-multiple: true

        - name: Download sklearn server image
          uses: actions/download-artifact@v4
          with:
            name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
            path: ./tmp

        - name: Download custom model grpc image
          uses: actions/download-artifact@v4
          with:
            name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.CUSTOM_MODEL_GRPC_IMG }}-${{ github.sha }}
            path: ./tmp

        - name: Download xgb server image
          uses: actions/download-artifact@v4
          with:
            name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
            path: ./tmp

        - name: Load docker images
          uses: ./.github/actions/load-docker-images
          with:
            directory: ./tmp

        - name: Install Poetry and version plugin
          run: ./test/scripts/gh-actions/setup-poetry.sh

        - name: Install KServe
          run: |
            ./test/scripts/gh-actions/setup-kserve.sh
        
            kubectl get pods -n kserve
            kubectl describe pods -n kserve

        - name: Run E2E tests
          timeout-minutes: 30
          run: |
            ./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation or explainer" "6"

        - name: Check system status
          if: always()
          run: |
            ./test/scripts/gh-actions/status-check.sh

  test-graph:
    runs-on: ubuntu-latest
    needs:
      [
        kserve-image-build,
        predictor-runtime-build,
        graph-tests-images-build,
      ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download graph artifacts
        uses: actions/download-artifact@v4
        with:
          path: ./tmp
          pattern: ${{ env.GRAPH_ARTIFACT_PREFIX }}-*
          merge-multiple: true

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download xgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests for graph
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "graph" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh


  test-path-based-routing:
    runs-on: ubuntu-latest
    needs:
      [
        kserve-image-build,
        predictor-runtime-build
      ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download xgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download lgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.LGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download transformer image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.TRANSFORMER_ARTIFACT_PREFIX }}-${{ env.IMAGE_TRANSFORMER_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Patch inferenceservice config
        run: |
          kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-ingress.yaml
          kubectl describe configmaps -n kserve inferenceservice-config

      - name: Run E2E tests with path-based routing
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "path_based_routing" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-qpext:
    runs-on: ubuntu-latest
    needs: [kserve-image-build, predictor-runtime-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Build queue proxy extension image
        run: |
          ./test/scripts/gh-actions/build-qpext-image.sh
          docker image ls

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh
      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh
          kubectl get pods -n kserve
          kubectl describe pods -n kserve
      - name: Patch qpext image
        run: |
          kubectl patch configmaps -n knative-serving config-deployment --patch '{"data": {"queue-sidecar-image": "kserve/qpext:${{ github.sha }}"}}'
          kubectl describe configmaps -n knative-serving config-deployment
      - name: Run queue proxy extension E2E tests
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-qpext-test.sh

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-with-helm:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - uses: ./.github/actions/free-up-disk-space
      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install Kserve from helm
        run: |
          ./test/scripts/gh-actions/setup-modelmesh-dep.sh
          ./test/scripts/gh-actions/setup-kserve-helm.sh
          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 20
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "helm"
          kubectl get pods -n kserve

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-raw:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup
        with:
          deployment-mode: "raw"

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download custom model grpc image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.CUSTOM_MODEL_GRPC_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download transformer image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.TRANSFORMER_ARTIFACT_PREFIX }}-${{ env.IMAGE_TRANSFORMER_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh "raw"

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "raw" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-kourier:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, graph-tests-images-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup
        with:
          network-layer: "kourier"

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Download graph artifacts
        uses: actions/download-artifact@v4
        with:
          path: ./tmp
          pattern: ${{ env.GRAPH_ARTIFACT_PREFIX }}-*
          merge-multiple: true

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download xgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Patch inferenceservice config to disable istio
        run: |
          kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-disable-istio.yaml
          kubectl describe configmaps -n kserve inferenceservice-config

      - name: Run E2E tests
        timeout-minutes: 30
        run: |
          export KSERVE_INGRESS_HOST_PORT=$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \
          --output=jsonpath="{.items[0].status.podIP}"):$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \
          --output=jsonpath="{.items[0].spec.containers[0].ports[0].containerPort}")

          ./test/scripts/gh-actions/run-e2e-tests.sh "kourier" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh "kourier"