e2e-test.yml

name: E2E Tests

on:
  pull_request:
    branches: [master, release*]
    paths:
      - "**"
      - "!.github/**"
      - "!docs/**"
      - "!**.md"
      - ".github/workflows/e2e-test.yml"
  workflow_dispatch:

env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  DOCKER_IMAGES_PATH: "/mnt/docker-images"
  DOCKER_REPO: "kserve"
  # artifact prefixes for bulk download
  PREDICTOR_ARTIFACT_PREFIX: "pred"
  EXPLAINER_ARTIFACT_PREFIX: "exp"
  TRANSFORMER_ARTIFACT_PREFIX: "trans"
  GRAPH_ARTIFACT_PREFIX: "graph"
  BASE_ARTIFACT_PREFIX: "base"
  # Controller images
  CONTROLLER_IMG: "kserve-controller"
  STORAGE_INIT_IMG: "storage-initializer"
  AGENT_IMG: "agent"
  ROUTER_IMG: "router"
  # Predictor runtime server images
  SKLEARN_IMG: "sklearnserver"
  XGB_IMG: "xgbserver"
  LGB_IMG: "lgbserver"
  PMML_IMG: "pmmlserver"
  PADDLE_IMG: "paddleserver"
  CUSTOM_MODEL_GRPC_IMG: "custom-model-grpc"
  HUGGINGFACE_IMG: "huggingfaceserver"
  # Explainer images
  ART_IMG: "art-explainer"
  # Transformer images
  IMAGE_TRANSFORMER_IMG: "image-transformer"
  IMAGE_TRANSFORMER_IMG_TAG: "kserve/image-transformer:${{ github.sha }}"
  CUSTOM_TRANSFORMER_GRPC_IMG: "custom-image-transformer-grpc"
  # Graph images
  SUCCESS_200_ISVC_IMG: "success-200-isvc"
  ERROR_404_ISVC_IMG: "error-404-isvc"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  kserve-image-build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build KServe images
        run: |
          sudo mkdir -p ${DOCKER_IMAGES_PATH}
          sudo chown -R $USER ${DOCKER_IMAGES_PATH}
          ./test/scripts/gh-actions/build-images.sh
          docker image ls
          sudo ls -lh ${DOCKER_IMAGES_PATH}

      - name: Upload controller image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.BASE_ARTIFACT_PREFIX }}-${{ env.CONTROLLER_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.CONTROLLER_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload agent image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.BASE_ARTIFACT_PREFIX }}-${{ env.AGENT_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.AGENT_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload storage initializer image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.BASE_ARTIFACT_PREFIX }}-${{ env.STORAGE_INIT_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.STORAGE_INIT_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload router image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.BASE_ARTIFACT_PREFIX }}-${{ env.ROUTER_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.ROUTER_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

  poetry-check:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Check poetry lock file consistency
        run: ./test/scripts/gh-actions/check-poetry-lockfile.sh

  predictor-runtime-build:
    runs-on: ubuntu-latest
    needs: [poetry-check]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build runtime server images
        run: |
          sudo mkdir -p ${DOCKER_IMAGES_PATH}
          sudo chown -R $USER ${DOCKER_IMAGES_PATH}
          ./test/scripts/gh-actions/build-server-runtimes.sh predictor,transformer
          docker image ls
          sudo ls -lh ${DOCKER_IMAGES_PATH}

      - name: Upload sklearn artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.SKLEARN_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload xgb server image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.XGB_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload lgb server image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.LGB_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.LGB_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload pmml server image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.PMML_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.PMML_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload paddle image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.PADDLE_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.PADDLE_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload image transformer image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.TRANSFORMER_ARTIFACT_PREFIX }}-${{ env.IMAGE_TRANSFORMER_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.IMAGE_TRANSFORMER_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload custom model grpc image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.CUSTOM_MODEL_GRPC_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.CUSTOM_MODEL_GRPC_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload custom model transformer grpc image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.TRANSFORMER_ARTIFACT_PREFIX }}-${{ env.CUSTOM_TRANSFORMER_GRPC_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.CUSTOM_TRANSFORMER_GRPC_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload huggingface image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.HUGGINGFACE_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.HUGGINGFACE_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

  explainer-runtime-build:
    runs-on: ubuntu-latest
    needs: [poetry-check]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build runtime server images
        run: |
          sudo mkdir -p ${DOCKER_IMAGES_PATH}
          sudo chown -R $USER ${DOCKER_IMAGES_PATH}
          ./test/scripts/gh-actions/build-server-runtimes.sh explainer
          docker image ls
          sudo ls -lh ${DOCKER_IMAGES_PATH}

      - name: Upload art explainer image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.EXPLAINER_ARTIFACT_PREFIX }}-${{ env.ART_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.ART_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

  graph-tests-images-build:
    runs-on: ubuntu-latest
    needs: [poetry-check]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build test images needed for graph tests
        run: |
          sudo mkdir -p ${DOCKER_IMAGES_PATH}
          sudo chown -R $USER ${DOCKER_IMAGES_PATH}
          ./test/scripts/gh-actions/build-graph-tests-images.sh
          docker image ls
          sudo ls -lh ${DOCKER_IMAGES_PATH}

      - name: Upload success_200_isvc predictor image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.GRAPH_ARTIFACT_PREFIX }}-${{ env.SUCCESS_200_ISVC_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.SUCCESS_200_ISVC_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

      - name: Upload error_404_isvc predictor image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.GRAPH_ARTIFACT_PREFIX }}-${{ env.ERROR_404_ISVC_IMG }}-${{ github.sha }}
          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.ERROR_404_ISVC_IMG }}-${{ github.sha }}
          compression-level: 0
          if-no-files-found: error

  test-predictor:
    runs-on: ubuntu-latest
    needs:
      [
        kserve-image-build,
        predictor-runtime-build,
      ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Download predictor artifacts
        uses: actions/download-artifact@v4
        with:
          path: ./tmp
          pattern: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-*
          merge-multiple: true

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run predictor E2E tests
        timeout-minutes: 40
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "predictor" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-transformer-explainer-mms:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Download transformer and explainer artifacts
        uses: actions/download-artifact@v4
        with:
          path: ./tmp
          pattern: +(${{ env.TRANSFORMER_ARTIFACT_PREFIX }}|${{ env.EXPLAINER_ARTIFACT_PREFIX }})-*
          merge-multiple: true

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download custom model grpc image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.CUSTOM_MODEL_GRPC_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download xgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh
          
          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation or explainer" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-graph:
    runs-on: ubuntu-latest
    needs:
      [
        kserve-image-build,
        predictor-runtime-build,
        graph-tests-images-build,
      ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download graph artifacts
        uses: actions/download-artifact@v4
        with:
          path: ./tmp
          pattern: ${{ env.GRAPH_ARTIFACT_PREFIX }}-*
          merge-multiple: true

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download xgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests for graph
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "graph" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh


  test-path-based-routing:
    runs-on: ubuntu-latest
    needs:
      [
        kserve-image-build,
        predictor-runtime-build
      ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download xgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download lgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.LGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download transformer image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.TRANSFORMER_ARTIFACT_PREFIX }}-${{ env.IMAGE_TRANSFORMER_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Patch inferenceservice config
        run: |
          kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-ingress.yaml
          kubectl describe configmaps -n kserve inferenceservice-config

      - name: Run E2E tests with path-based routing
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "path_based_routing" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-qpext:
    runs-on: ubuntu-latest
    needs: [kserve-image-build, predictor-runtime-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Build queue proxy extension image
        run: |
          ./test/scripts/gh-actions/build-qpext-image.sh
          docker image ls

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh
      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh
          kubectl get pods -n kserve
          kubectl describe pods -n kserve
      - name: Patch qpext image
        run: |
          kubectl patch configmaps -n knative-serving config-deployment --patch '{"data": {"queue-sidecar-image": "kserve/qpext:${{ github.sha }}"}}'
          kubectl describe configmaps -n knative-serving config-deployment
      - name: Run queue proxy extension E2E tests
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-qpext-test.sh

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-with-helm:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - uses: ./.github/actions/free-up-disk-space
      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install Kserve from helm
        run: |
          ./test/scripts/gh-actions/setup-modelmesh-dep.sh
          ./test/scripts/gh-actions/setup-kserve-helm.sh
          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 20
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "helm"
          kubectl get pods -n kserve

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-raw:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup
        with:
          deployment-mode: "raw"

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download xgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download custom model grpc image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.CUSTOM_MODEL_GRPC_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download transformer image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.TRANSFORMER_ARTIFACT_PREFIX }}-${{ env.IMAGE_TRANSFORMER_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh "raw"

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "raw" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh

  test-kourier:
    runs-on: ubuntu-latest
    needs:
      [kserve-image-build, predictor-runtime-build, graph-tests-images-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup
        with:
          network-layer: "kourier"

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Download graph artifacts
        uses: actions/download-artifact@v4
        with:
          path: ./tmp
          pattern: ${{ env.GRAPH_ARTIFACT_PREFIX }}-*
          merge-multiple: true

      - name: Download sklearn server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Download xgb server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Patch inferenceservice config to disable istio
        run: |
          kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-disable-istio.yaml
          kubectl describe configmaps -n kserve inferenceservice-config

      - name: Run E2E tests
        timeout-minutes: 30
        run: |
          export KSERVE_INGRESS_HOST_PORT=$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \
          --output=jsonpath="{.items[0].status.podIP}"):$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \
          --output=jsonpath="{.items[0].spec.containers[0].ports[0].containerPort}")

          ./test/scripts/gh-actions/run-e2e-tests.sh "kourier" "6"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh "kourier"

  test-llm:
    runs-on: ubuntu-latest
    needs:
      [ kserve-image-build, predictor-runtime-build]
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.21"
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Setup Minikube
        uses: ./.github/actions/minikube-setup

      - name: KServe dependency setup
        uses: ./.github/actions/kserve-dep-setup

      - name: Download base images
        uses: ./.github/actions/base-download

      - name: Download huggingface server image
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.HUGGINGFACE_IMG }}-${{ github.sha }}
          path: ./tmp

      - name: Load docker images
        uses: ./.github/actions/load-docker-images
        with:
          directory: ./tmp

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Install KServe
        run: |
          ./test/scripts/gh-actions/setup-kserve.sh

          kubectl get pods -n kserve
          kubectl describe pods -n kserve

      - name: Run E2E tests
        timeout-minutes: 30
        run: |
          ./test/scripts/gh-actions/run-e2e-tests.sh "llm" "2"

      - name: Check system status
        if: always()
        run: |
          ./test/scripts/gh-actions/status-check.sh