Skip to content

Commit

Permalink
fix merge_async_iterators usage for vLLM>0.5.4
Browse files Browse the repository at this point in the history
  • Loading branch information
dtrifiro committed Aug 7, 2024
1 parent 8a7c5a2 commit 09290e6
Showing 1 changed file with 12 additions and 3 deletions.
15 changes: 12 additions & 3 deletions src/vllm_tgis_adapter/grpc/grpc_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,9 +257,18 @@ async def Generate(
)

# TODO handle cancellation
result_generator: AsyncIterator[tuple[int, RequestOutput]] = (
merge_async_iterators(*generators)
)
result_generator: AsyncIterator[tuple[int, RequestOutput]]

kwargs = {}
if "is_cancelled" in inspect.signature(merge_async_iterators).parameters:
# vllm > 0.5.4

async def is_cancelled() -> bool:
return context.cancelled()

kwargs["is_cancelled"] = is_cancelled

result_generator = merge_async_iterators(*generators, **kwargs)

resp_options = request.params.response
responses: list = [None] * request_count
Expand Down

0 comments on commit 09290e6

Please sign in to comment.