Skip to content

Commit

Permalink
add vllm-tgis-adapter layer
Browse files Browse the repository at this point in the history
  • Loading branch information
dtrifiro committed Sep 12, 2024
1 parent 02fde86 commit d7a74df
Showing 1 changed file with 19 additions and 0 deletions.
19 changes: 19 additions & 0 deletions Dockerfile.rocm.ubi
Original file line number Diff line number Diff line change
Expand Up @@ -244,3 +244,22 @@ USER 2000

# Set the entrypoint
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]


FROM vllm-openai as vllm-grpc-adapter

USER root

RUN --mount=type=cache,target=/root/.cache/pip \
pip install vllm-tgis-adapter==0.4.0

ENV GRPC_PORT=8033 \
PORT=8000 \
# As an optimization, vLLM disables logprobs when using spec decoding by
# default, but this would be unexpected to users of a hosted model that
# happens to have spec decoding
# see: https://github.com/vllm-project/vllm/pull/6485
DISABLE_LOGPROBS_DURING_SPEC_DECODING=false

USER 2000
ENTRYPOINT ["python3", "-m", "vllm_tgis_adapter", "--uvicorn-log-level=warning"]

0 comments on commit d7a74df

Please sign in to comment.