From eca7cdf66975d4e08ce3579e828d73b9c6b904dc Mon Sep 17 00:00:00 2001 From: Sivanantham Chinnaiyan Date: Sun, 11 Feb 2024 22:57:02 +0530 Subject: [PATCH] Use docker-container driver Signed-off-by: Sivanantham Chinnaiyan --- .github/actions/base-download/action.yml | 34 ++--- .github/workflows/e2e-test.yml | 132 ++++++++---------- .../gh-actions/build-graph-tests-images.sh | 8 +- test/scripts/gh-actions/build-images.sh | 16 +-- .../gh-actions/build-server-runtimes.sh | 40 +++--- 5 files changed, 104 insertions(+), 126 deletions(-) diff --git a/.github/actions/base-download/action.yml b/.github/actions/base-download/action.yml index 118709987be..8bcf41c0936 100644 --- a/.github/actions/base-download/action.yml +++ b/.github/actions/base-download/action.yml @@ -10,28 +10,14 @@ runs: path: ./tmp pattern: ${{ env.BASE_ARTIFACT_PREFIX }}-* merge-multiple: true -# -# - name: Load base images -# run: | -# pushd ./tmp >> /dev/null -# docker image load -i sflj -# - name: Download controller image -# uses: ishworkh/docker-image-artifact-download@v1 -# with: -# image: "kserve/kserve-controller:${{ github.sha }}" -# -# - name: Download agent image -# uses: ishworkh/docker-image-artifact-download@v1 -# with: -# image: "kserve/agent:${{ github.sha }}" -# -# - name: Download storage initializer image -# uses: ishworkh/docker-image-artifact-download@v1 -# with: -# image: "kserve/storage-initializer:${{ github.sha }}" -# -# - name: Download router image -# uses: ishworkh/docker-image-artifact-download@v1 -# with: -# image: "kserve/router:${{ github.sha }}" + - name: Load base images + shell: bash + run: | + files=$(find ./tmp -maxdepth 1 -type f) + for file in ${files[@]};do + echo "Loading image $(basename ${file})" + docker image load -i ${file} + done + rm -rf ./tmp + docker image ls diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 18d838404b5..2f612e4bc1b 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -15,6 +15,7 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCKER_IMAGES_PATH: /mnt/docker-images DOCKER_REPO: kserve + # artifact prefixes for bulk download PREDICTOR_ARTIFACT_PREFIX: pred EXPLAINER_ARTIFACT_PREFIX: exp TRANSFORMER_ARTIFACT_PREFIX: trans @@ -42,7 +43,6 @@ env: SUCCESS_200_ISVC_IMG: success-200-isvc ERROR_404_ISVC_IMG: error-404-isvc - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -53,6 +53,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build KServe images run: | sudo mkdir -p ${DOCKER_IMAGES_PATH} @@ -117,6 +120,9 @@ jobs: - name: Free-up disk space uses: ./.github/actions/free-up-disk-space + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build runtime server images run: | sudo mkdir -p ${DOCKER_IMAGES_PATH} @@ -195,6 +201,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build runtime server images run: | sudo mkdir -p ${DOCKER_IMAGES_PATH} @@ -228,6 +237,9 @@ jobs: - name: Free-up disk space uses: ./.github/actions/free-up-disk-space + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build test images needed for graph tests run: | sudo mkdir -p ${DOCKER_IMAGES_PATH} @@ -292,36 +304,15 @@ jobs: pattern: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-* merge-multiple: true - -# - name: Download sklearn server image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/sklearnserver:${{ github.sha }}" -# -# - name: Download xgb server image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/xgbserver:${{ github.sha }}" -# -# - name: Download lgb server image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/lgbserver:${{ github.sha }}" -# -# - name: Download pmml image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/pmmlserver:${{ github.sha }}" -# -# - name: Download paddle image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/paddleserver:${{ github.sha }}" -# -# - name: Download custom model grpc image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/custom-model-grpc:${{ github.sha }}" + - name: Load predictor images + run: | + files=$(find ./tmp -maxdepth 1 -type f) + for file in ${files[@]};do + echo "Loading image $(basename ${file})" + docker image load -i ${file} + done + rm -rf ./tmp + docker image ls - name: Install Poetry and version plugin run: ./test/scripts/gh-actions/setup-poetry.sh @@ -365,44 +356,36 @@ jobs: uses: actions/download-artifact@v4 with: path: ./tmp - pattern: ${{ env.TRANSFORMER_ARTIFACT_PREFIX }}-*|${{ env.EXPLAINER_ARTIFACT_PREFIX }}-* + pattern: (${{ env.TRANSFORMER_ARTIFACT_PREFIX }}|${{ env.EXPLAINER_ARTIFACT_PREFIX }})-* merge-multiple: true - name: Download sklearn server image uses: actions/download-artifact@v4 with: - name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }} + name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }} path: ./tmp - name: Download custom model grpc image - uses: ishworkh/docker-image-artifact-download@v1 + uses: actions/download-artifact@v4 with: - image: "kserve/custom-model-grpc:${{ github.sha }}" - -# - name: Download transformer image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/image-transformer:${{ github.sha }}" -# -# - name: Download custom transformer grpc image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/custom-image-transformer-grpc:${{ github.sha }}" + name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.CUSTOM_MODEL_GRPC_IMG }}-${{ github.sha }} + path: ./tmp - name: Download xgb server image - uses: ishworkh/container-image-artifact-download@v1.0.0 + uses: actions/download-artifact@v4 with: - image: "kserve/xgbserver:${{ github.sha }}" - -# - name: Download alibi image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/alibi-explainer:${{ github.sha }}" -# -# - name: Download art explainer image -# uses: ishworkh/container-image-artifact-download@v1.0.0 -# with: -# image: "kserve/art-explainer:${{ github.sha }}" + name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }} + path: ./tmp + + - name: Load docker images + run: | + files=$(find ./tmp -maxdepth 1 -type f) + for file in ${files[@]};do + echo "Loading image $(basename ${file})" + docker image load -i ${file} + done + rm -rf ./tmp + docker image ls - name: Install Poetry and version plugin run: ./test/scripts/gh-actions/setup-poetry.sh @@ -458,25 +441,34 @@ jobs: - name: KServe dependency setup uses: ./.github/actions/kserve-dep-setup - - name: Download sklearn server image - uses: ishworkh/container-image-artifact-download@v1.0.0 + - name: Download graph artifacts + uses: actions/download-artifact@v4 with: - image: "kserve/sklearnserver:${{ github.sha }}" + path: ./tmp + pattern: ${{ env.GRAPH_ARTIFACT_PREFIX }}-* + merge-multiple: true - - name: Download xgb server image - uses: ishworkh/docker-image-artifact-download@v1 + - name: Download sklearn server image + uses: actions/download-artifact@v4 with: - image: "kserve/xgbserver:${{ github.sha }}" + name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.SKLEARN_IMG }}-${{ github.sha }} + path: ./tmp - - name: Download success_200_isvc predictor image - uses: ishworkh/container-image-artifact-download@v1.0.0 + - name: Download xgb server image + uses: actions/download-artifact@v4 with: - image: "kserve/success-200-isvc:${{ github.sha }}" + name: ${{ env.PREDICTOR_ARTIFACT_PREFIX }}-${{ env.XGB_IMG }}-${{ github.sha }} + path: ./tmp - - name: Download error_404_isvc predictor image - uses: ishworkh/container-image-artifact-download@v1.0.0 - with: - image: "kserve/error-404-isvc:${{ github.sha }}" + - name: Load docker images + run: | + files=$(find ./tmp -maxdepth 1 -type f) + for file in ${files[@]};do + echo "Loading image $(basename ${file})" + docker image load -i ${file} + done + rm -rf ./tmp + docker image ls - name: Install Poetry and version plugin run: ./test/scripts/gh-actions/setup-poetry.sh diff --git a/test/scripts/gh-actions/build-graph-tests-images.sh b/test/scripts/gh-actions/build-graph-tests-images.sh index e2c9cd3f043..d469d224c71 100755 --- a/test/scripts/gh-actions/build-graph-tests-images.sh +++ b/test/scripts/gh-actions/build-graph-tests-images.sh @@ -26,12 +26,12 @@ ERROR_404_ISVC_IMG_TAG=${DOCKER_REPO}/${ERROR_404_ISVC_IMG}:${GITHUB_SHA} pushd python >/dev/null echo "Building success_200_isvc image" -docker buildx build -t "${SUCCESS_200_ISVC_IMG_TAG}" -f success_200_isvc.Dockerfile . -docker image save -o "${DOCKER_IMAGES_PATH}/${SUCCESS_200_ISVC_IMG}-${GITHUB_SHA}" "${SUCCESS_200_ISVC_IMG_TAG}" +docker buildx build -t "${SUCCESS_200_ISVC_IMG_TAG}" -f success_200_isvc.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${SUCCESS_200_ISVC_IMG}-${GITHUB_SHA}",compression-level=0 . echo "Done building success_200_isvc image" echo "Building error_404_isvc image" -docker buildx build -t "${ERROR_404_ISVC_IMG_TAG}" -f error_404_isvc.Dockerfile . -docker image save -o "${DOCKER_IMAGES_PATH}/${ERROR_404_ISVC_IMG}-${GITHUB_SHA}" "${ERROR_404_ISVC_IMG_TAG}" +docker buildx build -t "${ERROR_404_ISVC_IMG_TAG}" -f error_404_isvc.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${ERROR_404_ISVC_IMG}-${GITHUB_SHA}",compression-level=0 . echo "Done building error_404_isvc image" popd echo "Done building images" diff --git a/test/scripts/gh-actions/build-images.sh b/test/scripts/gh-actions/build-images.sh index 9e6398d4412..c7a23e5fa8a 100755 --- a/test/scripts/gh-actions/build-images.sh +++ b/test/scripts/gh-actions/build-images.sh @@ -29,21 +29,21 @@ AGENT_IMG_TAG=${DOCKER_REPO}/${AGENT_IMG}:${GITHUB_SHA} ROUTER_IMG_TAG=${DOCKER_REPO}/${ROUTER_IMG}:${GITHUB_SHA} echo "Building Kserve controller image" -docker buildx build . -t "${CONTROLLER_IMG_TAG}" -docker image save -o "${DOCKER_IMAGES_PATH}/${CONTROLLER_IMG}-${GITHUB_SHA}" "${CONTROLLER_IMG_TAG}" +docker buildx build . -t "${CONTROLLER_IMG_TAG}" \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${CONTROLLER_IMG}-${GITHUB_SHA}",compression-level=0 echo "Building agent image" -docker buildx build -f agent.Dockerfile . -t "${AGENT_IMG_TAG}" -docker image save -o "${DOCKER_IMAGES_PATH}/${AGENT_IMG}-${GITHUB_SHA}" "${AGENT_IMG_TAG}" +docker buildx build -f agent.Dockerfile . -t "${AGENT_IMG_TAG}" \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${AGENT_IMG}-${GITHUB_SHA}",compression-level=0 echo "Building router image" -docker buildx build -f router.Dockerfile . -t "${ROUTER_IMG_TAG}" -docker image save -o "${DOCKER_IMAGES_PATH}/${ROUTER_IMG}-${GITHUB_SHA}" "${ROUTER_IMG_TAG}" +docker buildx build -f router.Dockerfile . -t "${ROUTER_IMG_TAG}" \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${ROUTER_IMG}-${GITHUB_SHA}",compression-level=0 pushd python >/dev/null echo "Building storage initializer" - docker buildx build -t "${STORAGE_INIT_IMG_TAG}" -f storage-initializer.Dockerfile . - docker image save -o "${DOCKER_IMAGES_PATH}/${STORAGE_INIT_IMG}-${GITHUB_SHA}" "${STORAGE_INIT_IMG_TAG}" + docker buildx build -f storage-initializer.Dockerfile . -t "${STORAGE_INIT_IMG_TAG}" \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${STORAGE_INIT_IMG}-${GITHUB_SHA}",compression-level=0 popd echo "Done building images" diff --git a/test/scripts/gh-actions/build-server-runtimes.sh b/test/scripts/gh-actions/build-server-runtimes.sh index 939cce023a0..fa973e7ed04 100755 --- a/test/scripts/gh-actions/build-server-runtimes.sh +++ b/test/scripts/gh-actions/build-server-runtimes.sh @@ -42,41 +42,41 @@ types=("$1") pushd python >/dev/null if [[ " ${types[*]} " =~ "predictor" ]]; then echo "Building Sklearn image" - docker buildx build -t "${SKLEARN_IMG_TAG}" -f sklearn.Dockerfile . - docker image save -o "${DOCKER_IMAGES_PATH}/${SKLEARN_IMG}-${GITHUB_SHA}" "${SKLEARN_IMG_TAG}" + docker buildx build -t "${SKLEARN_IMG_TAG}" -f sklearn.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${SKLEARN_IMG}-${GITHUB_SHA}",compression-level=0 . echo "Building XGB image" - docker buildx build -t "${XGB_IMG_TAG}" -f xgb.Dockerfile . - docker image save -o "${DOCKER_IMAGES_PATH}/${XGB_IMG}-${GITHUB_SHA}" "${XGB_IMG_TAG}" + docker buildx build -t "${XGB_IMG_TAG}" -f xgb.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${XGB_IMG}-${GITHUB_SHA}",compression-level=0 . echo "Building LGB image" - docker buildx build -t "${LGB_IMG_TAG}" -f lgb.Dockerfile . - docker image save -o "${DOCKER_IMAGES_PATH}/${LGB_IMG}-${GITHUB_SHA}" "${LGB_IMG_TAG}" + docker buildx build -t "${LGB_IMG_TAG}" -f lgb.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${LGB_IMG}-${GITHUB_SHA}",compression-level=0 . echo "Building PMML image" - docker buildx build -t "${PMML_IMG_TAG}" -f pmml.Dockerfile . - docker image save -o "${DOCKER_IMAGES_PATH}/${PMML_IMG}-${GITHUB_SHA}" "${PMML_IMG_TAG}" + docker buildx build -t "${PMML_IMG_TAG}" -f pmml.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${PMML_IMG}-${GITHUB_SHA}",compression-level=0 . echo "Building Paddle image" - docker buildx build -t "${PADDLE_IMG_TAG}" -f paddle.Dockerfile . - docker image save -o "${DOCKER_IMAGES_PATH}/${PADDLE_IMG}-${GITHUB_SHA}" "${PADDLE_IMG_TAG}" + docker buildx build -t "${PADDLE_IMG_TAG}" -f paddle.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${PADDLE_IMG}-${GITHUB_SHA}",compression-level=0 . echo "Building Custom model gRPC image" - docker buildx build -t "${CUSTOM_MODEL_GRPC_IMG_TAG}" -f custom_model_grpc.Dockerfile . - docker image save -o "${DOCKER_IMAGES_PATH}/${CUSTOM_MODEL_GRPC_IMG}-${GITHUB_SHA}" "${CUSTOM_MODEL_GRPC_IMG_TAG}" + docker buildx build -t "${CUSTOM_MODEL_GRPC_IMG_TAG}" -f custom_model_grpc.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${CUSTOM_MODEL_GRPC_IMG}-${GITHUB_SHA}",compression-level=0 . echo "Building image transformer gRPC image" - docker buildx build -t "${CUSTOM_TRANSFORMER_GRPC_IMG_TAG}" -f custom_transformer_grpc.Dockerfile . - docker image save -o "${DOCKER_IMAGES_PATH}/${CUSTOM_TRANSFORMER_GRPC_IMG}-${GITHUB_SHA}" "${CUSTOM_TRANSFORMER_GRPC_IMG_TAG}" + docker buildx build -t "${CUSTOM_TRANSFORMER_GRPC_IMG_TAG}" -f custom_transformer_grpc.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${CUSTOM_TRANSFORMER_GRPC_IMG}-${GITHUB_SHA}",compression-level=0 . fi if [[ " ${types[*]} " =~ "explainer" ]]; then echo "Building Alibi image" - docker buildx build -t "${ALIBI_IMG_TAG}" -f alibiexplainer.Dockerfile . - docker save -o "${DOCKER_IMAGES_PATH}/${ALIBI_IMG}-${GITHUB_SHA}" "${ALIBI_IMG_TAG}" + docker buildx build -t "${ALIBI_IMG_TAG}" -f alibiexplainer.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${ALIBI_IMG}-${GITHUB_SHA}",compression-level=0 . echo "Building ART explainer image" - docker buildx build -t "${ART_IMG_TAG}" -f artexplainer.Dockerfile . - docker save -o "${DOCKER_IMAGES_PATH}/${ART_IMG}-${GITHUB_SHA}" "${ART_IMG_TAG}" + docker buildx build -t "${ART_IMG_TAG}" -f artexplainer.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${ART_IMG}-${GITHUB_SHA}",compression-level=0 . fi if [[ " ${types[*]} " =~ "transformer" ]]; then echo "Building Image transformer image" - docker buildx build -t "${IMAGE_TRANSFORMER_IMG_TAG}" -f custom_transformer.Dockerfile . - docker save -o "${DOCKER_IMAGES_PATH}/${IMAGE_TRANSFORMER_IMG}-${GITHUB_SHA}" "${IMAGE_TRANSFORMER_IMG_TAG}" + docker buildx build -t "${IMAGE_TRANSFORMER_IMG_TAG}" -f custom_transformer.Dockerfile \ + -o type=docker,dest="${DOCKER_IMAGES_PATH}/${IMAGE_TRANSFORMER_IMG}-${GITHUB_SHA}",compression-level=0 . fi popd