Skip to content
Merlin CI Workflow

Add run_pyfunc_model() to build and run pyfunc model locally #1901

Sign in to view logs

Add run_pyfunc_model() to build and run pyfunc model locally

Add run_pyfunc_model() to build and run pyfunc model locally #1901

Workflow file for this run

.github/workflows/merlin.yml at 7b70a40
name: Merlin CI Workflow
on:
push:
branches:
- main
tags:
- v*
pull_request:
env:
ARTIFACT_RETENTION_DAYS: 7
DOCKER_BUILDKIT: 1
DOCKER_REGISTRY: ghcr.io
GO_VERSION: "1.20"
jobs:
create-version:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.create_version.outputs.version }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- id: create_version
name: Create version string
run: |
# Strip git ref prefix from version
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
[ "$VERSION" == "main" ] && VERSION=$(git describe --tags --always --first-parent)
# Strip "v" prefix
[[ "${VERSION}" == "v"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
# If it's pull request the version string is prefixed by 0.0.0-
[ ${{ github.event_name}} == "pull_request" ] && VERSION="0.0.0-${{ github.event.pull_request.head.sha }}"
echo ${VERSION}
echo "::set-output name=version::${VERSION}"
test-batch-predictor:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10"]
env:
PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ matrix.python-version }}-
- uses: actions/cache@v3
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-python-${{ matrix.python-version }}-pipenv-batch-predictor
- name: Install dependencies
working-directory: ./python/batch-predictor
run: |
pip install pipenv==2023.7.23
make setup
- name: Run batch-predictor test
working-directory: ./python/batch-predictor
run: make unit-test
test-pyfunc-server:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10"]
env:
PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ matrix.python-version }}-
- uses: actions/cache@v3
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-python-${{ matrix.python-version }}-pipenv-pyfunc-server
- name: Install dependencies
working-directory: ./python/pyfunc-server
run: |
pip install pipenv==2023.7.23
make setup
- name: Run pyfunc-server test
working-directory: ./python/pyfunc-server
run: make test
test-python-sdk:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10"]
env:
PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ matrix.python-version }}-
- uses: actions/cache@v3
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-python-${{ matrix.python-version }}-pipenv-python-sdk
- name: Install dependencies
working-directory: ./python/sdk
run: |
pip install pipenv==2023.7.23
make setup
- name: Unit test Python SDK
env:
E2E_USE_GOOGLE_OAUTH: false
working-directory: ./python/sdk
run: make unit-test
lint-api:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
# Ensure the same version as the one defined in Makefile
version: v1.51.2
working-directory: api
test-api:
runs-on: ubuntu-latest
services:
postgres:
image: postgres:12.4
env:
POSTGRES_DB: ${{ secrets.DB_NAME }}
POSTGRES_USER: ${{ secrets.DB_USERNAME }}
POSTGRES_PASSWORD: ${{ secrets.DB_PASSWORD }}
ports:
- 5432:5432
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Install dependencies
run: |
make setup
make init-dep-api
- name: Test API files
env:
POSTGRES_HOST: localhost
POSTGRES_DB: ${{ secrets.DB_NAME }}
POSTGRES_USER: ${{ secrets.DB_USERNAME }}
POSTGRES_PASSWORD: ${{ secrets.DB_PASSWORD }}
run: make it-test-api-ci
build-ui:
runs-on: ubuntu-latest
steps:
- name: Checkout to the target branch
uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: 16
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "::set-output name=dir::$(yarn cache dir)"
- name: Cache YARN
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Cache Node Modules
uses: actions/cache@v2
with:
path: ui/node_modules
key: |
${{ runner.os }}-modules-${{ hashFiles('ui/yarn.lock') }}
restore-keys: ${{ runner.os }}-modules-
- name: Install dependencies
run: make init-dep-ui
- name: Lint UI files
run: make lint-ui
- name: Test UI files
run: make test-ui
- name: Build UI static files
run: make build-ui
- name: Publish UI Artifact
uses: actions/upload-artifact@v2
with:
name: merlin-ui-dist
path: ui/build/
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-api:
runs-on: ubuntu-latest
services:
postgres:
image: postgres:12.4
env:
POSTGRES_DB: ${{ secrets.DB_NAME }}
POSTGRES_USER: ${{ secrets.DB_USERNAME }}
POSTGRES_PASSWORD: ${{ secrets.DB_PASSWORD }}
ports:
- 5432:5432
needs:
- create-version
- build-ui
- test-api
steps:
- uses: actions/checkout@v2
- name: Download UI Dist
uses: actions/download-artifact@v2
with:
name: merlin-ui-dist
path: ui/build
- name: Build API Docker
run: docker build -t merlin:${{ needs.create-version.outputs.version }} -f Dockerfile .
- name: Save API Docker
run: docker image save --output merlin.${{ needs.create-version.outputs.version }}.tar merlin:${{ needs.create-version.outputs.version }}
- name: Publish API Docker Artifact
uses: actions/upload-artifact@v2
with:
name: merlin.${{ needs.create-version.outputs.version }}.tar
path: merlin.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-batch-predictor-base:
runs-on: ubuntu-latest
needs:
- create-version
- test-batch-predictor
env:
DOCKER_IMAGE_TAG: "ghcr.io/${{ github.repository }}/merlin-pyspark-base:${{ needs.create-version.outputs.version }}"
steps:
- uses: actions/checkout@v2
- name: Build Batch Predictor Base Docker
run: docker build -t ${{ env.DOCKER_IMAGE_TAG }} -f python/batch-predictor/docker/base.Dockerfile python
- name: Save Batch Predictor Base Docker
run: docker image save --output merlin-pyspark-base.${{ needs.create-version.outputs.version }}.tar ${{ env.DOCKER_IMAGE_TAG }}
- name: Publish Batch Predictor Base Docker Artifact
uses: actions/upload-artifact@v2
with:
name: merlin-pyspark-base.${{ needs.create-version.outputs.version }}.tar
path: merlin-pyspark-base.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-pyfunc-server-base:
runs-on: ubuntu-latest
needs:
- create-version
- test-pyfunc-server
env:
DOCKER_REGISTRY: ghcr.io
DOCKER_IMAGE_TAG: "ghcr.io/${{ github.repository }}/merlin-pyfunc-base:${{ needs.create-version.outputs.version }}"
steps:
- uses: actions/checkout@v2
- name: Build Pyfunc Server Base Docker
run: docker build -t ${{ env.DOCKER_IMAGE_TAG }} -f python/pyfunc-server/docker/base.Dockerfile python
- name: Save Pyfunc Server Base Docker
run: docker image save --output merlin-pyfunc-base.${{ needs.create-version.outputs.version }}.tar ${{ env.DOCKER_IMAGE_TAG }}
- name: Publish Pyfunc Server Base Docker Artifact
uses: actions/upload-artifact@v2
with:
name: merlin-pyfunc-base.${{ needs.create-version.outputs.version }}.tar
path: merlin-pyfunc-base.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-transformer:
runs-on: ubuntu-latest
needs:
- create-version
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Install dependencies
run: make init-dep-api
- name: Build Standard Transformer
run: make build-transformer
- name: Build Standard Transformer Docker
run: docker build -t merlin-transformer:${{ needs.create-version.outputs.version }} -f transformer.Dockerfile .
- name: Save Standard Transformer Docker
run: docker image save --output merlin-transformer.${{ needs.create-version.outputs.version }}.tar merlin-transformer:${{ needs.create-version.outputs.version }}
- name: Publish Standard Transformer Docker Artifact
uses: actions/upload-artifact@v2
with:
name: merlin-transformer.${{ needs.create-version.outputs.version }}.tar
path: merlin-transformer.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
build-inference-logger:
runs-on: ubuntu-latest
needs:
- create-version
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
- name: Build Inference Logger
run: make build-inference-logger
- name: Build Inference Logger Docker
run: docker build -t merlin-logger:${{ needs.create-version.outputs.version }} -f inference-logger.Dockerfile .
- name: Save Inference Logger Docker
run: docker image save --output merlin-logger.${{ needs.create-version.outputs.version }}.tar merlin-logger:${{ needs.create-version.outputs.version }}
- name: Publish Inference Logger Docker Artifact
uses: actions/upload-artifact@v2
with:
name: merlin-logger.${{ needs.create-version.outputs.version }}.tar
path: merlin-logger.${{ needs.create-version.outputs.version }}.tar
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
e2e-test:
runs-on: ubuntu-latest
needs:
- build-api
- build-transformer
- create-version
env:
K3D_CLUSTER: merlin-cluster
LOCAL_REGISTRY_PORT: 12345
LOCAL_REGISTRY: "dev.localhost"
INGRESS_HOST: "127.0.0.1.nip.io"
MERLIN_CHART_VERSION: 0.13.4
E2E_PYTHON_VERSION: "3.10.6"
K3S_VERSION: v1.26.7-k3s1
steps:
- uses: actions/checkout@v2
with:
path: merlin
- uses: actions/setup-python@v4
with:
python-version: ${{ env.E2E_PYTHON_VERSION }}
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-3.10-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-3.10-
- uses: actions/cache@v3
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-python-3.10-pipenv-python-sdk
- name: Download k3d
run: |
curl --silent --fail https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.6.0 bash
- name: Create Test Cluster
run: |
k3d registry create $LOCAL_REGISTRY --port $LOCAL_REGISTRY_PORT
k3d cluster create $K3D_CLUSTER --image rancher/k3s:${K3S_VERSION} --port 80:80@loadbalancer \
--k3s-arg '--disable=traefik,metrics-server@server:*' \
--k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@server:0' \
--k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@server:0' \
--k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@agent:*' \
--k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@agent:*'
- name: Setup cluster
working-directory: merlin/scripts/e2e
run: ./setup-cluster.sh merlin-cluster
- name: Download API Docker Artifact
uses: actions/download-artifact@v2
with:
name: merlin.${{ needs.create-version.outputs.version }}.tar
- name: Download Standard Transformer Docker Artifact
uses: actions/download-artifact@v2
with:
name: merlin-transformer.${{ needs.create-version.outputs.version }}.tar
- name: Publish images to k3d registry
run: |
# Merlin API
docker image load --input merlin.${{ needs.create-version.outputs.version }}.tar
docker tag merlin:${{ needs.create-version.outputs.version }} ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }}/merlin:${{ needs.create-version.outputs.version }}
k3d image import ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }}/merlin:${{ needs.create-version.outputs.version }} -c merlin-cluster
# Standard Transformer
docker image load --input merlin-transformer.${{ needs.create-version.outputs.version }}.tar
docker tag merlin-transformer:${{ needs.create-version.outputs.version }} ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }}/merlin-transformer:${{ needs.create-version.outputs.version }}
k3d image import ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }}/merlin-transformer:${{ needs.create-version.outputs.version }} -c merlin-cluster
- name: Deploy merlin and mlp
working-directory: merlin/scripts/e2e
run: ./deploy-merlin.sh ${{ env.INGRESS_HOST }} ${{ env.LOCAL_REGISTRY }}:${{ env.LOCAL_REGISTRY_PORT }} ${{ needs.create-version.outputs.version }} ${{ github.ref }} ${{ env.MERLIN_CHART_VERSION }}
- name: Prune docker image to make some space
run: docker image prune --all --force
- name: Run E2E Test
timeout-minutes: 30
id: run-e2e-test
working-directory: merlin/scripts/e2e
run: ./run-e2e.sh ${{ env.INGRESS_HOST }} ${{ env.E2E_PYTHON_VERSION }}
- name: "Debug"
if: always()
continue-on-error: true
working-directory: merlin/scripts/e2e
run: ./debug-e2e.sh
release:
uses: ./.github/workflows/release.yml
needs:
- create-version
- build-api
- build-batch-predictor-base
- build-pyfunc-server-base
- test-python-sdk
- e2e-test
with:
version: ${{ needs.create-version.outputs.version }}
secrets:
pypi_username: ${{ secrets.PYPI_USERNAME }}
pypi_password: ${{ secrets.PYPI_PASSWORD }}
ghcr_token: ${{ secrets.GITHUB_TOKEN }}