custom kserve runtime

Co-Authored-By: Wesley Gifford <[email protected]>

services/inference/Dockerfile

@@ -0,0 +1,44 @@
+# based on https://github.com/opendatahub-io/caikit-tgis-serving/blob/main/Dockerfile
+
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest as builder
+
+RUN microdnf -y update && \
+    microdnf -y install \
+        git shadow-utils python3.11-pip python-wheel && \
+    pip3.11 install --no-cache-dir --upgrade pip wheel && \
+    microdnf clean all
+
+ENV POETRY_VIRTUALENVS_IN_PROJECT=1
+
+COPY tsfmservices* /tsfmservices/tsfmservices
+COPY pyproject.toml /tsfmservices/
+COPY poetry.lock /tsfmservices/
+WORKDIR /tsfmservices
+# RUN python3.11 -m venv /tsfmservices/.venv
+#RUN source /tsfmservices/.venv/bin/activate && \
+# pip3.11 install --no-cache-dir ".[services]"
+RUN pip3.11 install poetry && poetry install
+
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest as deploy
+RUN microdnf -y update && \
+    microdnf -y install \
+        shadow-utils python3.11 && \
+    microdnf clean all
+
+WORKDIR /tsfmservices
+
+COPY --from=builder /tsfmservices/ /tsfmservices/
+
+ENV VIRTUAL_ENV=/tsfmservices/.venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV HF_HOME=/tmp
+
+RUN groupadd --system tsfmservices --gid 1001 && \
+    adduser --system --uid 1001 --gid 0 --groups tsfmservices \
+    --create-home --home-dir /tsfmservices --shell /sbin/nologin \
+    --comment "TSFMServices User" tsfmservices
+
+USER tsfmservices
+
+# CMD ["python",  "-m", "tsfmservices.inference.main"]
+CMD ["python", "-m", "uvicorn","tsfmservices.inference.main:app", "--host", "0.0.0.0", "--port", "8000" ]

services/inference/Makefile

@@ -0,0 +1,30 @@
+CONTAINER_BUILDER ?= docker
+
+# starts the inference service (used mainly for test cases)
+start_service_local:
+	python -m tsfmservices.inference.main &
+	sleep 10
+stop_service_local:
+	pkill  -f 'python.*tsfmservices.*'
+	sleep 10
+
+image:
+	$(CONTAINER_BUILDER) build -t tsfmservices -f Dockerfile .
+
+start_service_image: image
+	$(CONTAINER_BUILDER) run -p 8000:8000 -d --rm --name tsfmserver tsfmservices
+	sleep 10
+stop_service_image:
+	$(CONTAINER_BUILDER) stop tsfmserver
+
+test_local: start_service_local
+	pytest tests
+	$(MAKE) stop_service_local
+
+test_image: start_service_image
+	pytest tests
+	$(MAKE) stop_service_image
+
+
+
+

services/inference/README.md

@@ -0,0 +1,76 @@
+# TSFM Services
+
+This component provides basic RESTful services for the IBM tsfm-granite
+class of timeseries foundation models. At present it can serve the following models:
+
+* https://huggingface.co/ibm-granite/granite-timeseries-ttm-v1
+* https://huggingface.co/ibm-granite/granite-timeseries-patchtst
+* https://huggingface.co/ibm-granite/granite-timeseries-patchtsmixer
+
+
+
+## Prerequisites:
+
+* GNU make
+* python 3.10 or 3.11
+* poetry (`pip install poetry`)
+* zsh or bash (zsh is preferred)
+
+_Note that our primary target environment for services deployment is x86_64 Linux. You may encounter hiccups if you try to use this on a different environment. If so, please
+file an issue. Some of our developers do use a Mac so you're likely to find a quick resolution. None of our developers use native Windows, however._
+
+## Known issues:
+
+* Use of pkill statements in Makefile may not work properly on Mac OS. This will be apparent if you have left over processs after running test related make targets. Please help us put OS-specific checks into our Makefile to handle these cases by filing a PR.
+
+### Installation
+
+```sh
+pip install poetry && poetry install --with dev
+```
+
+### Testing using a local server instance
+
+```sh
+make test_local
+```
+
+### Creating an image
+
+_You must have either docker or podman installed on your system for this to
+work. You must also have proper permissions on your system to build images._
+
+```sh
+CONTAINER_BUILDER=<docker|podmain> make image
+# e.g, CONTAINER_BUILDER=docker make image
+```
+
+After a successful build you should have a local image named `tsfmservices:latest`
+
+```sh
+(py311) ➜  tsfm-services git:(revised-build-system) ✗ docker images | grep tsfmservices | head -n 1
+tsfmservices                                             latest               df592dcb0533   46 seconds ago      1.49GB
+# some of the numeric and hash values on your machine could be different
+```
+
+### Testing using the built image
+
+```sh
+make test_image
+```
+
+### Viewing the OpenAPI 3.x specification and swagger page
+
+```sh
+make start_local_server
+```
+
+Then open your browser to http://127.0.0.1:8000/docs, click on the `/openapi.json` link at the top of this page.
+
+To stop the server run:
+
+```sh
+# may not work properly on a Mac
+# if not, kill the uvicorn process manually.
+make stop_local_server
+```