Skip to content
Merlin CI Workflow

Update Merlin to use Kserve 0.11.0 #1447

Sign in to view logs

Update Merlin to use Kserve 0.11.0

Update Merlin to use Kserve 0.11.0 #1447

Workflow file for this run

.github/workflows/merlin.yml at 17879c5
name: Merlin CI Workflow
on:
push:
branches:
- main
tags:
- v*
pull_request:
env:
ARTIFACT_RETENTION_DAYS: 7
DOCKER_BUILDKIT: 1
GO_VERSION: "1.20"
jobs:
create-version:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.create_version.outputs.version }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- id: create_version
name: Create version string
run: |
# Strip git ref prefix from version
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
[ "$VERSION" == "main" ] && VERSION=$(git describe --tags --always --first-parent)
# Strip "v" prefix
[[ "${VERSION}" == "v"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
# If it's pull request the version string is prefixed by 0.0.0-
[ ${{ github.event_name}} == "pull_request" ] && VERSION="0.0.0-${{ github.event.pull_request.head.sha }}"
echo ${VERSION}
echo "::set-output name=version::${VERSION}"
test-batch-predictor:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10"]
env:
PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ matrix.python-version }}-
- uses: actions/cache@v3
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-python-${{ matrix.python-version }}-pipenv-batch-predictor
- name: Install dependencies
working-directory: ./python/batch-predictor
run: |
pip install pipenv==2023.7.23
make setup
- name: Run batch-predictor test
working-directory: ./python/batch-predictor
run: make unit-test
test-pyfunc-server:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10"]
env:
PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ matrix.python-version }}-
- uses: actions/cache@v3
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-python-${{ matrix.python-version }}-pipenv-pyfunc-server
- name: Install dependencies
working-directory: ./python/pyfunc-server
run: |
pip install pipenv==2023.7.23
make setup
- name: Run pyfunc-server test
working-directory: ./python/pyfunc-server
run: make test
test-python-sdk:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10"]
env:
PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ matrix.python-version }}-
- uses: actions/cache@v3
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-python-${{ matrix.python-version }}-pipenv-python-sdk
- name: Install dependencies
working-directory: ./python/sdk
run: |
pip install pipenv==2023.7.23
make setup
- name: Unit test Python SDK
env:
E2E_USE_GOOGLE_OAUTH: false
working-directory: ./python/sdk
run: make unit-test
lint-api:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
# Ensure the same version as the one defined in Makefile
version: v1.51.2
working-directory: api
test-api:
runs-on: ubuntu-latest
services:
postgres:
image: postgres:12.4
env:
POSTGRES_DB: ${{ secrets.DB_NAME }}
POSTGRES_USER: ${{ secrets.DB_USERNAME }}
POSTGRES_PASSWORD: ${{ secrets.DB_PASSWORD }}
ports:
- 5432:5432
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Install dependencies
run: |
make setup
make init-dep-api
- name: Test API files
env:
POSTGRES_HOST: localhost
POSTGRES_DB: ${{ secrets.DB_NAME }}
POSTGRES_USER: ${{ secrets.DB_USERNAME }}
POSTGRES_PASSWORD: ${{ secrets.DB_PASSWORD }}
run: make it-test-api-ci
build-ui:
runs-on: ubuntu-latest
steps:
- name: Checkout to the target branch
uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: 16
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "::set-output name=dir::$(yarn cache dir)"
- name: Cache YARN
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Cache Node Modules
uses: actions/cache@v2
with:
path: ui/node_modules
key: |
${{ runner.os }}-modules-${{ hashFiles('ui/yarn.lock') }}
restore-keys: ${{ runner.os }}-modules-
- name: Install dependencies
run: make init-dep-ui
- name: Lint UI files
run: make lint-ui
- name: Test UI files
run: make test-ui
- name: Build UI static files
run: make build-ui
- name: Publish UI Artifact
uses: actions/upload-artifact@v2
with:
name: merlin-ui-dist
path: ui/build/
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-api:
runs-on: ubuntu-latest
services:
postgres:
image: postgres:12.4
env:
POSTGRES_DB: ${{ secrets.DB_NAME }}
POSTGRES_USER: ${{ secrets.DB_USERNAME }}
POSTGRES_PASSWORD: ${{ secrets.DB_PASSWORD }}
ports:
- 5432:5432
needs:
- build-ui
- create-version
steps:
- uses: actions/checkout@v2
- name: Download UI Dist
uses: actions/download-artifact@v2
with:
name: merlin-ui-dist
path: ui/build
- name: Build API Docker
run: docker build -t merlin:${{ needs.create-version.outputs.version }} -f Dockerfile .
- name: Save API Docker
run: docker image save --output merlin.${{ needs.create-version.outputs.version }}.tar merlin:${{ needs.create-version.outputs.version }}
- name: Publish API Docker Artifact
uses: actions/upload-artifact@v2
with:
name: merlin.${{ needs.create-version.outputs.version }}.tar
path: merlin.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-batch-predictor:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["37", "38", "39", "310"]
needs:
- create-version
steps:
- uses: actions/checkout@v2
- name: Build Batch Predictor Docker py${{ matrix.python-version }}
run: docker build -t merlin-pyspark-base-py${{ matrix.python-version }}:${{ needs.create-version.outputs.version }} --build-arg PYTHON_VERSION=${{ matrix.python-version }} -f python/batch-predictor/docker/base.Dockerfile python
- name: Save Batch Predictor Docker
run: docker image save --output merlin-pyspark-base-py${{ matrix.python-version }}.${{ needs.create-version.outputs.version }}.tar merlin-pyspark-base-py${{ matrix.python-version }}:${{ needs.create-version.outputs.version }}
- name: Publish Batch Predictor Docker Artifact
uses: actions/upload-artifact@v2
with:
name: merlin-pyspark-base-py${{ matrix.python-version }}.${{ needs.create-version.outputs.version }}.tar
path: merlin-pyspark-base-py${{ matrix.python-version }}.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-pyfunc-server:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["37", "38", "39", "310"]
needs:
- create-version
steps:
- uses: actions/checkout@v2
- name: Build Pyfunc Server Docker py${{ matrix.python-version }}
run: docker build -t merlin-pyfunc-base-py${{ matrix.python-version }}:${{ needs.create-version.outputs.version }} --build-arg PYTHON_VERSION=${{ matrix.python-version }} -f python/pyfunc-server/docker/base.Dockerfile python
- name: Save Pyfunc Server Docker py${{ matrix.python-version }}
run: docker image save --output merlin-pyfunc-base-py${{ matrix.python-version }}.${{ needs.create-version.outputs.version }}.tar merlin-pyfunc-base-py${{ matrix.python-version }}:${{ needs.create-version.outputs.version }}
- name: Publish Pyfunc Server Docker Artifact py${{ matrix.python-version }}
uses: actions/upload-artifact@v2
with:
name: merlin-pyfunc-base-py${{ matrix.python-version }}.${{ needs.create-version.outputs.version }}.tar
path: merlin-pyfunc-base-py${{ matrix.python-version }}.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-transformer:
runs-on: ubuntu-latest
needs:
- create-version
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Install dependencies
run: make init-dep-api
- name: Build Standard Transformer
run: make build-transformer
- name: Build Standard Transformer Docker
run: docker build -t merlin-transformer:${{ needs.create-version.outputs.version }} -f transformer.Dockerfile .
- name: Save Standard Transformer Docker
run: docker image save --output merlin-transformer.${{ needs.create-version.outputs.version }}.tar merlin-transformer:${{ needs.create-version.outputs.version }}
- name: Publish Standard Transformer Docker Artifact
uses: actions/upload-artifact@v2
with:
name: merlin-transformer.${{ needs.create-version.outputs.version }}.tar
path: merlin-transformer.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-inference-logger:
runs-on: ubuntu-latest
needs:
- create-version
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
- name: Build Inference Logger
run: make build-inference-logger
- name: Build Inference Logger Docker
run: docker build -t merlin-logger:${{ needs.create-version.outputs.version }} -f inference-logger.Dockerfile .
- name: Save Inference Logger Docker
run: docker image save --output merlin-logger.${{ needs.create-version.outputs.version }}.tar merlin-logger:${{ needs.create-version.outputs.version }}
- name: Publish Inference Logger Docker Artifact
uses: actions/upload-artifact@v2
with:
name: merlin-logger.${{ needs.create-version.outputs.version }}.tar
path: merlin-logger.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
e2e-test:
runs-on: ubuntu-latest
needs:
- build-api
- build-transformer
- create-version
env:
K3D_CLUSTER: merlin-cluster
LOCAL_REGISTRY_PORT: 12345
LOCAL_REGISTRY: "dev.localhost"
INGRESS_HOST: "127.0.0.1.nip.io"
MERLIN_CHART_VERSION: 0.11.4
E2E_PYTHON_VERSION: "3.10.6"
K3S_VERSION: v1.26.7-k3s1
steps:
- uses: actions/checkout@v2
with:
path: merlin
- uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/setup-python@v4
with:
python-version: ${{ env.E2E_PYTHON_VERSION }}
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ matrix.python-version }}-
- uses: actions/cache@v3
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-python-${{ matrix.python-version }}-pipenv-${{ hashFiles('Pipfile.lock') }}
- name: Download k3d
run: |
curl --silent --fail https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.4.1 bash
- name: Create Test Cluster
run: |
k3d registry create $LOCAL_REGISTRY --port $LOCAL_REGISTRY_PORT
k3d cluster create $K3D_CLUSTER --image rancher/k3s:${K3S_VERSION} --k3s-arg '--disable=traefik,metrics-server@server:*' --port 80:80@loadbalancer
- name: Setup cluster
working-directory: merlin/scripts/e2e
run: ./setup-cluster.sh merlin-cluster ${{ env.INGRESS_HOST }}
- name: Download API Docker Artifact
uses: actions/download-artifact@v2
with:
name: merlin.${{ needs.create-version.outputs.version }}.tar
- name: Download Standard Transformer Docker Artifact
uses: actions/download-artifact@v2
with:
name: merlin-transformer.${{ needs.create-version.outputs.version }}.tar
- name: Publish images to k3d registry
run: |
# Merlin API
docker image load --input merlin.${{ needs.create-version.outputs.version }}.tar
docker tag merlin:${{ needs.create-version.outputs.version }} ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }}/merlin:${{ needs.create-version.outputs.version }}
k3d image import ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }}/merlin:${{ needs.create-version.outputs.version }} -c merlin-cluster
# Standard Transformer
docker image load --input merlin-transformer.${{ needs.create-version.outputs.version }}.tar
docker tag merlin-transformer:${{ needs.create-version.outputs.version }} ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }}/merlin-transformer:${{ needs.create-version.outputs.version }}
k3d image import ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }}/merlin-transformer:${{ needs.create-version.outputs.version }} -c merlin-cluster
- name: Deploy merlin and mlp
working-directory: merlin/scripts/e2e
run: ./deploy-merlin.sh ${{ env.INGRESS_HOST }} ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }} ${{ needs.create-version.outputs.version }} ${{ github.ref }} ${{ env.MERLIN_CHART_VERSION }}
- name: Run E2E Test
timeout-minutes: 30
id: run-e2e-test
working-directory: merlin/scripts/e2e
run: ./run-e2e.sh ${{ env.INGRESS_HOST }} ${{ env.E2E_PYTHON_VERSION }}
- name: "Debug"
if: always()
continue-on-error: true
run: |
echo "::group::Get nodes"
kubectl get nodes
echo "::endgroup::"
echo "::group::Get all events"
kubectl get events -A
echo "::endgroup::"
echo "::group::Get cert-manager logs"
kubectl logs deploy/cert-manager -n cert-manager
echo "::endgroup::"
echo "::group::Get KServe pods"
kubectl get pods -n kserve -o wide
kubectl get pods -n kserve -o yaml
echo "::endgroup::"
echo "::group::KServe log"
kubectl logs deployment/kserve-controller-manager -n kserve -c manager
echo "::endgroup::"
echo "::group::KServe events"
kubectl get events -n kserve
echo "::endgroup::"
echo "::group::Get Knative pods"
kubectl get pods -n knative-serving -o wide
kubectl get pods -n knative-serving -o yaml
echo "::endgroup::"
echo "::group::Knative log"
kubectl logs deploy/controller -n knative-serving
echo "::endgroup::"
echo "::group::Get Minio pods"
kubectl get pods -n minio-operator -o wide
kubectl get pods -n minio-tenant -o wide
kubectl get pods -n minio-tenant -o yaml
kubectl get pvc -n minio-tenant -o yaml
echo "::endgroup::"
echo "::group::Get deployments in caraml namespace"
kubectl get deployment -n caraml
echo "::endgroup::"
echo "::group::Describe pods in caraml namespace"
kubectl describe pods -n caraml
echo "::endgroup::"
echo "::group::Merlin log"
kubectl logs deploy/merlin -n caraml -c merlin
echo "::endgroup::"
echo "::group::Get deployment in merlin-e2e namespace"
kubectl get deployment -n merlin-e2e
echo "::endgroup::"
echo "::group::Describe deployment in merlin-e2e namespace"
kubectl describe deployment -n merlin-e2e
echo "::endgroup::"
echo "::group::Get pods in merlin-e2e namespace"
kubectl get pod -n merlin-e2e -o yaml
echo "::endgroup::"
echo "::group::Describe pods in merlin-e2e namespace"
kubectl describe pod -n merlin-e2e
echo "::endgroup::"
echo "::group::Get events in merlin-e2e namespace"
kubectl get events -n merlin-e2e
echo "::endgroup::"
echo "::group::Get Inference Service in merlin-e2e namespace"
kubectl get isvc -n merlin-e2e -o yaml
echo "::endgroup::"
echo "::group::Models log"
kubectl logs -l component=predictor -c kserve-container -n merlin-e2e
echo "::endgroup::"
echo "::group::Queue proxy log"
kubectl logs -l component=predictor -c queue-proxy -n merlin-e2e
echo "::endgroup::"
echo "::group::Transformers log"
kubectl logs -l component=transformer -c kserve-container -n merlin-e2e
echo "::endgroup::"
release:
uses: ./.github/workflows/release.yml
needs:
- create-version
- e2e-test
- test-api
- test-python-sdk
- test-pyfunc-server
- test-batch-predictor
with:
version: ${{ needs.create-version.outputs.version }}
secrets:
pypi_username: ${{ secrets.PYPI_USERNAME }}
pypi_password: ${{ secrets.PYPI_PASSWORD }}
ghcr_token: ${{ secrets.GITHUB_TOKEN }}