fvt.yml

name: FVT

on:
  workflow_dispatch:
  pull_request:
    branches:
      - main
      - 'release-[0-9].[0-9]+'
    paths:
      - '**'
      - '!.github/**'
      - '!.tekton/**'
      - '!**.md'
      - '!docs/**'
      - '.github/workflows/fvt.yml'

jobs:
  fvt:
    runs-on: ubuntu-latest
    steps:
      - name: Free up disk space
        run: |
          # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
          # du -sh /*        2> /dev/null | sort -rh  2> /dev/null | head
          # du -h -d2 /usr   2> /dev/null | sort -rh  2> /dev/null | head
          echo "Check free disk space before cleanup."
          df -h
          echo "Removing non-essential tools and libraries."
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/share/boost
          # delete libraries for Android (12G), PowerShell (1.3G), Swift (1.7G)
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /usr/local/share/powershell
          sudo rm -rf /usr/share/swift
          echo "Check free disk space after cleanup."
          df -h

      - name: Checkout
        uses: actions/checkout@v3

      - name: Setup go
        uses: actions/setup-go@v4
        with:
          go-version: '1.19'

      - name: Start Minikube
        uses: medyagh/setup-minikube@v0.0.13
        id: minikube
        with:
          minikube-version: 1.31.0
          container-runtime: docker
          kubernetes-version: v1.26.1
          cpus: max
          memory: max

      - name: Check pods
        run: |
          sleep 30
          kubectl get pods -n kube-system

      - name: Set controller image tag
        run: echo "IMAGE_TAG=$(date +'%Y%m%dT%H%M%S%Z')" >> $GITHUB_ENV

      - name: Update configs
        # Update the image tag and reduce some resource request amounts to allow FVTs to run
        # on reduced resource environments. The RollingUpdate strategy for Runtime deployments
        # is also adjusted for these environments.
        # Disable the torchserve ServingRuntime for now (insufficient resources to run them all).
        run: |
          sed -i 's/newTag:.*$/newTag: '"${{ env.IMAGE_TAG }}"'/' config/manager/kustomization.yaml
          sed -i '0,/cpu:.*$/s/cpu:.*$/cpu: 100m/' \
            config/default/config-defaults.yaml \
            config/runtimes/mlserver-1.x.yaml \
            config/runtimes/triton-2.x.yaml \
            config/runtimes/ovms-1.x.yaml
          sed -i 's/memory:.*$/memory: 512Mi/g' \
            config/runtimes/mlserver-1.x.yaml \
            config/runtimes/triton-2.x.yaml \
            config/runtimes/ovms-1.x.yaml
          sed -i 's/maxSurge:.*$/maxSurge: 0/' config/internal/base/deployment.yaml.tmpl
          sed -i 's/maxUnavailable:.*$/maxUnavailable: 100%/' config/internal/base/deployment.yaml.tmpl
          echo -e '\n  disabled: true' >> config/runtimes/torchserve-0.x.yaml

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build Controller image
        env:
          DOCKER_BUILDKIT: 1
        run: |
          eval $(minikube -p minikube docker-env)
          make build.develop
          ./scripts/build_docker.sh --target runtime --tag ${{ env.IMAGE_TAG }}

      - name: Install ModelMesh Serving
        run: |
          kubectl create ns modelmesh-serving
          ./scripts/install.sh --namespace modelmesh-serving --fvt --dev-mode-logging

      - name: Prune docker images
        run: |
          echo "Pruning docker images on GH action runner node"
          docker image prune -a -f
          docker system df
          df -h

      - name: Pre-pull runtime images
        run: |
          eval $(minikube -p minikube docker-env)
          docker pull nvcr.io/nvidia/tritonserver:23.04-py3
          docker pull seldonio/mlserver:1.3.2
          docker pull openvino/model_server:2022.2
          # docker pull pytorch/torchserve:0.7.1-cpu
          docker pull kserve/modelmesh:latest
          docker pull kserve/modelmesh-minio-dev-examples:latest
          docker pull kserve/modelmesh-minio-examples:latest
          docker pull kserve/modelmesh-runtime-adapter:latest
          docker pull kserve/rest-proxy:latest

      - name: Check installation
        run: |
          eval $(minikube -p minikube docker-env)
          docker images
          kubectl get pods
          kubectl get clusterservingruntimes

      - name: Run FVTs
        run: |
          go install github.com/onsi/ginkgo/v2/ginkgo
          export PATH=/root/go/bin/:$PATH
          export NAMESPACE=modelmesh-serving
          export NAMESPACESCOPEMODE=false
          make fvt