Skip to content

Commit

Permalink
add vllm-tgis-adapter layer
Browse files Browse the repository at this point in the history
  • Loading branch information
dtrifiro committed Sep 16, 2024
1 parent 93c04f3 commit 66984d4
Showing 1 changed file with 19 additions and 0 deletions.
19 changes: 19 additions & 0 deletions Dockerfile.rocm.ubi
Original file line number Diff line number Diff line change
Expand Up @@ -236,3 +236,22 @@ USER 2000

# Set the entrypoint
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]


FROM vllm-openai as vllm-grpc-adapter

USER root

RUN --mount=type=cache,target=/root/.cache/pip \
pip install vllm-tgis-adapter==0.4.0

ENV GRPC_PORT=8033 \
PORT=8000 \
# As an optimization, vLLM disables logprobs when using spec decoding by
# default, but this would be unexpected to users of a hosted model that
# happens to have spec decoding
# see: https://github.com/vllm-project/vllm/pull/6485
DISABLE_LOGPROBS_DURING_SPEC_DECODING=false

USER 2000
ENTRYPOINT ["python3", "-m", "vllm_tgis_adapter", "--uvicorn-log-level=warning"]

0 comments on commit 66984d4

Please sign in to comment.