test: Increase initial time to stabilize for FVTs #112
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: FVT | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - 'release-[0-9].[0-9]+' | |
| paths: | |
| - '**' | |
| - '!.github/**' | |
| - '!.tekton/**' | |
| - '!**.md' | |
| - '!docs/**' | |
| - '.github/workflows/fvt.yml' | |
| jobs: | |
| fvt: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Free up disk space | |
| run: | | |
| # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| # du -sh /* 2> /dev/null | sort -rh 2> /dev/null | head | |
| # du -h -d2 /usr 2> /dev/null | sort -rh 2> /dev/null | head | |
| echo "Check free disk space before cleanup." | |
| df -h | |
| echo "Removing non-essential tools and libraries." | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/share/boost | |
| # delete libraries for Android (12G), PowerShell (1.3G), Swift (1.7G) | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/local/share/powershell | |
| sudo rm -rf /usr/share/swift | |
| echo "Check free disk space after cleanup." | |
| df -h | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: '1.19' | |
| - name: Start Minikube | |
| uses: medyagh/[email protected] | |
| id: minikube | |
| with: | |
| minikube-version: 1.31.0 | |
| container-runtime: docker | |
| kubernetes-version: v1.26.1 | |
| cpus: max | |
| memory: max | |
| - name: Check pods | |
| run: | | |
| sleep 30 | |
| kubectl get pods -n kube-system | |
| - name: Set controller image tag | |
| run: echo "IMAGE_TAG=$(date +'%Y%m%dT%H%M%S%Z')" >> $GITHUB_ENV | |
| - name: Update configs | |
| # Update the image tag and reduce some resource request amounts to allow FVTs to run | |
| # on reduced resource environments. The RollingUpdate strategy for Runtime deployments | |
| # is also adjusted for these environments. | |
| # Disable the torchserve ServingRuntime for now (insufficient resources to run them all). | |
| run: | | |
| sed -i 's/newTag:.*$/newTag: '"${{ env.IMAGE_TAG }}"'/' config/manager/kustomization.yaml | |
| sed -i '0,/cpu:.*$/s/cpu:.*$/cpu: 100m/' \ | |
| config/default/config-defaults.yaml \ | |
| config/runtimes/mlserver-1.x.yaml \ | |
| config/runtimes/triton-2.x.yaml \ | |
| config/runtimes/ovms-1.x.yaml | |
| sed -i 's/memory:.*$/memory: 512Mi/g' \ | |
| config/runtimes/mlserver-1.x.yaml \ | |
| config/runtimes/triton-2.x.yaml \ | |
| config/runtimes/ovms-1.x.yaml | |
| sed -i 's/maxSurge:.*$/maxSurge: 0/' config/internal/base/deployment.yaml.tmpl | |
| sed -i 's/maxUnavailable:.*$/maxUnavailable: 100%/' config/internal/base/deployment.yaml.tmpl | |
| echo -e '\n disabled: true' >> config/runtimes/torchserve-0.x.yaml | |
| - name: Setup Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build Controller image | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| run: | | |
| eval $(minikube -p minikube docker-env) | |
| make build.develop | |
| ./scripts/build_docker.sh --target runtime --tag ${{ env.IMAGE_TAG }} | |
| - name: Install ModelMesh Serving | |
| run: | | |
| kubectl create ns modelmesh-serving | |
| ./scripts/install.sh --namespace modelmesh-serving --fvt --dev-mode-logging | |
| - name: Prune docker images | |
| run: | | |
| echo "Pruning docker images on GH action runner node" | |
| docker image prune -a -f | |
| docker system df | |
| df -h | |
| - name: Pre-pull runtime images | |
| run: | | |
| eval $(minikube -p minikube docker-env) | |
| docker pull nvcr.io/nvidia/tritonserver:23.04-py3 | |
| docker pull seldonio/mlserver:1.3.2 | |
| docker pull openvino/model_server:2022.2 | |
| # docker pull pytorch/torchserve:0.7.1-cpu | |
| docker pull kserve/modelmesh:latest | |
| docker pull kserve/modelmesh-minio-dev-examples:latest | |
| docker pull kserve/modelmesh-minio-examples:latest | |
| docker pull kserve/modelmesh-runtime-adapter:latest | |
| docker pull kserve/rest-proxy:latest | |
| - name: Check installation | |
| run: | | |
| eval $(minikube -p minikube docker-env) | |
| docker images | |
| kubectl get pods | |
| kubectl get clusterservingruntimes | |
| - name: Run FVTs | |
| run: | | |
| go install github.com/onsi/ginkgo/v2/ginkgo | |
| export PATH=/root/go/bin/:$PATH | |
| export NAMESPACE=modelmesh-serving | |
| export NAMESPACESCOPEMODE=false | |
| make fvt |