Enchance ci env

.github/actions/free-up-disk-space/action.yml

@@ -1,5 +1,5 @@
 name: 'Free-up disk space action'
-description: 'Removes non-essential tools, libraries and cached files from GitHub action runner node'
+description: 'Removes non-essential tools, libraries and cached files from GitHub action runner node and changes the docker data directory to /mnt/docker'
 
 runs:
   using: "composite"
@@ -8,7 +8,7 @@ runs:
       shell: bash
       run: |
         echo "Disk usage before cleanup:"
-        df -h
+        df -hT
 
         # remove non-essential tools and libraries, see:
         # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
@@ -23,12 +23,30 @@ runs:
         sudo rm -rf /usr/share/swift
 
         echo "Disk usage after cleanup:"
-        df -h
+        df -hT
 
     - name: Prune docker images
       shell: bash
       run: |
         echo "Pruning docker images"
         docker image prune -a -f
         docker system df
-        df -h
+        df -hT
+
+    - name: Move docker data directory
+      shell: bash
+      run: |
+        echo "Stopping docker service ..."
+        sudo systemctl stop docker
+        DOCKER_DEFAULT_ROOT_DIR=/var/lib/docker
+        DOCKER_ROOT_DIR=/mnt/docker
+        echo "Moving ${DOCKER_DEFAULT_ROOT_DIR} -> ${DOCKER_ROOT_DIR}"
+        sudo mv ${DOCKER_DEFAULT_ROOT_DIR} ${DOCKER_ROOT_DIR}
+        echo "Creating symlink ${DOCKER_DEFAULT_ROOT_DIR} -> ${DOCKER_ROOT_DIR}"
+        sudo ln -s ${DOCKER_ROOT_DIR} ${DOCKER_DEFAULT_ROOT_DIR}
+        echo "$(sudo ls -l ${DOCKER_DEFAULT_ROOT_DIR})"
+        echo "Starting docker service ..."
+        sudo systemctl daemon-reload
+        sudo systemctl start docker
+        echo "Docker service status:"
+        sudo systemctl --no-pager -l -o short status docker

.github/workflows/e2e-test.yml

@@ -221,6 +221,11 @@ jobs:
         with:
           image: "kserve/pmmlserver:${{ github.sha }}"
 
+      - name: Download paddle image
+        uses: ishworkh/docker-image-artifact-download@v1
+        with:
+          image: "kserve/paddleserver:${{ github.sha }}"
+
       - name: Download transformer image
         uses: ishworkh/docker-image-artifact-download@v1
         with:
@@ -249,12 +254,12 @@ jobs:
       - name: Run all E2E tests except graph
         timeout-minutes: 60
         run: |
-          ./test/scripts/gh-actions/run-e2e-tests.sh "fast or pmml" "2"
+          ./test/scripts/gh-actions/run-e2e-tests.sh "fast or pmml or slow" "6"
 
       - name: Run E2E tests for graph
         timeout-minutes: 120
         run: |
-          ./test/scripts/gh-actions/run-e2e-tests.sh "graph" "2"
+          ./test/scripts/gh-actions/run-e2e-tests.sh "graph" "6"
 
       - name: Check system status
         if: always()
@@ -334,70 +339,36 @@ jobs:
       - name: Run E2E tests with path-based routing
         timeout-minutes: 50
         run: |
-          ./test/scripts/gh-actions/run-e2e-tests.sh "fast" "2"
+          ./test/scripts/gh-actions/run-e2e-tests.sh "fast" "6"
 
       - name: Check system status
         if: always()
         run: |
           ./test/scripts/gh-actions/status-check.sh
 
-  test-slow:
-    runs-on: ubuntu-latest
-    needs:
-      [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v4
-        with:
-          go-version: "1.21"
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-
-      - uses: ./.github/actions/free-up-disk-space
-      - uses: ./.github/actions/minikube-setup
-      - uses: ./.github/actions/kserve-dep-setup
-      - uses: ./.github/actions/base-download
-
-      - name: Download sklearn server image
-        uses: ishworkh/docker-image-artifact-download@v1
-        with:
-          image: "kserve/sklearnserver:${{ github.sha }}"
-
-      - name: Download xgb server image
-        uses: ishworkh/docker-image-artifact-download@v1
-        with:
-          image: "kserve/xgbserver:${{ github.sha }}"
-
-      - name: Download lgb server image
-        uses: ishworkh/docker-image-artifact-download@v1
-        with:
-          image: "kserve/lgbserver:${{ github.sha }}"
-
-      - name: Download paddle image
-        uses: ishworkh/docker-image-artifact-download@v1
-        with:
-          image: "kserve/paddleserver:${{ github.sha }}"
-
-      - name: Install Poetry and version plugin
-        run: ./test/scripts/gh-actions/setup-poetry.sh
-
-      - name: Install KServe
-        run: |
-          ./test/scripts/gh-actions/setup-kserve.sh
-
-          kubectl get pods -n kserve
-          kubectl describe pods -n kserve
-
-      - name: Run E2E tests
-        timeout-minutes: 90
-        run: |
-          ./test/scripts/gh-actions/run-e2e-tests.sh "slow" "2"
-
-      - name: Check system status
-        if: always()
-        run: |
-          ./test/scripts/gh-actions/status-check.sh
+#  test-slow:
+#    runs-on: ubuntu-latest
+#    needs:
+#      [kserve-image-build, predictor-runtime-build, explainer-runtime-build]
+#    steps:
+#      - uses: actions/checkout@v3
+#      - uses: actions/setup-go@v4
+#        with:
+#          go-version: "1.21"
+#      - uses: actions/setup-python@v4
+#        with:
+#          python-version: "3.9"
+#
+#      - uses: ./.github/actions/free-up-disk-space
+#      - uses: ./.github/actions/minikube-setup
+#      - uses: ./.github/actions/kserve-dep-setup
+#      - uses: ./.github/actions/base-download
+#
+#
+#      - name: Check system status
+#        if: always()
+#        run: |
+#          ./test/scripts/gh-actions/status-check.sh
 
   test-explainer:
     runs-on: ubuntu-latest
@@ -445,7 +416,7 @@ jobs:
       - name: Run E2E tests
         timeout-minutes: 40
         run: |
-          ./test/scripts/gh-actions/run-e2e-tests.sh "explainer" "2"
+          ./test/scripts/gh-actions/run-e2e-tests.sh "explainer" "6"
 
       - name: Check system status
         if: always()
@@ -498,7 +469,7 @@ jobs:
       - name: Run E2E tests
         timeout-minutes: 40
         run: |
-          ./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation" "2"
+          ./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation" "6"
 
       - name: Check system status
         if: always()
@@ -604,7 +575,7 @@ jobs:
       - name: Run E2E tests
         timeout-minutes: 40
         run: |
-          ./test/scripts/gh-actions/run-e2e-tests.sh "grpc" "2"
+          ./test/scripts/gh-actions/run-e2e-tests.sh "grpc" "6"
 
       - name: Check system status
         if: always()
@@ -691,7 +662,7 @@ jobs:
       - name: Run E2E tests
         timeout-minutes: 40
         run: |
-          ./test/scripts/gh-actions/run-e2e-tests.sh "raw" "2"
+          ./test/scripts/gh-actions/run-e2e-tests.sh "raw" "6"
 
       - name: Check system status
         if: always()
@@ -760,7 +731,7 @@ jobs:
           --output=jsonpath="{.items[0].status.podIP}"):$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \
           --output=jsonpath="{.items[0].spec.containers[0].ports[0].containerPort}")
 
-          ./test/scripts/gh-actions/run-e2e-tests.sh "kourier" "2"
+          ./test/scripts/gh-actions/run-e2e-tests.sh "kourier" "6"
 
           kubectl get pods -n kserve
 

.github/workflows/python-test.yml

@@ -36,7 +36,12 @@ jobs:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Free-up disk space
+        uses: ./.github/actions/free-up-disk-space
+
       - name: Set up Python ${{ matrix.python-version }}
         id: setup-python
         uses: actions/setup-python@v4
@@ -46,12 +51,19 @@ jobs:
       - name: Install Poetry and version plugin
         run: ./test/scripts/gh-actions/setup-poetry.sh
 
+      - name: Configure Poetry cache
+        run: |
+          sudo mkdir -p /mnt/poetry
+          # change permission so that poetry can install without sudo
+          sudo chown -R $USER /mnt/poetry
+          poetry config cache-dir /mnt/poetry
+
       # load poetry cache if cache exists
       - name: Load poetry cache
         id: cached-poetry-dependencies
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pypoetry
+          path: /mnt/poetry
           key: poetry-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
       # ----------------------------------------Kserve Unit Tests--------------------------------------------------------
@@ -76,7 +88,7 @@ jobs:
         run: |
           cd python
           source kserve/.venv/bin/activate
-          pytest --cov=kserve ./kserve
+          pytest --cov=kserve -n 4 ./kserve
 
       # ----------------------------------------Sklearn Server Unit Tests------------------------------------------------
       # load cached sklearn venv if cache exists
@@ -225,24 +237,37 @@ jobs:
       # ----------------------------------------Huggingface Server Unit Tests------------------------------------------------
       # load cached huggingface venv if cache exists
       - name: Load cached huggingface venv
+        if: ${{ !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
         id: huggingface-dependencies
         uses: actions/cache@v3
         with:
-          path: python/huggingfaceserver/.venv
+          path: /mnt/python/huggingfaceserver-venv
           key: huggingface-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve/poetry.lock', '**/huggingfaceserver/poetry.lock') }}
         # install huggingface server dependencies if cache does not exist
+      - name: Configure poetry for huggingface server
+        if: ${{ !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
+        run: |
+          poetry config virtualenvs.path /mnt/python/huggingfaceserver-venv
+          poetry config virtualenvs.in-project false
       - name: Install huggingface dependencies
-        if: steps.cached-huggingface-dependencies.outputs.cache-hit != 'true'
+        if: ${{ steps.cached-huggingface-dependencies.outputs.cache-hit != 'true' && !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
         run: |
+          sudo mkdir -p /mnt/python/huggingfaceserver-venv
+          # change permission so that poetry can install without sudo
+          sudo chown -R $USER /mnt/python/huggingfaceserver-venv
           cd python/huggingfaceserver
           make install_dependencies
       - name: Install huggingface server
+        if: ${{ !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
         run: |
           cd python/huggingfaceserver
           make dev_install
       - name: Test huggingfaceserver
+        if: ${{ !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
         run: |
-          cd python
-          source huggingfaceserver/.venv/bin/activate
-          pytest --cov=huggingfaceserver ./huggingfaceserver
+          cd python/huggingfaceserver
+          poetry run -- pytest --cov=huggingfaceserver
 
+      - name: Free space after tests
+        run: |
+          df -hT