From 968d5e9651ca28b90498e282e3095f9654ceebe0 Mon Sep 17 00:00:00 2001 From: yehoshua Date: Sat, 4 May 2024 18:39:21 +0300 Subject: [PATCH] fix dependency injection --- vllm/entrypoints/llm.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 9ebb18179d53..c81e4e8bedd0 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -224,8 +224,9 @@ def _add_request( multi_modal_data: Optional[MultiModalData] = None, ) -> None: request_id = str(next(self.request_counter)) + truncate_prompt_tokens = sampling_params.truncate_prompt_tokens prompt, prompt_token_ids = self._validate_prompt( - prompt, prompt_token_ids, sampling_params) + prompt, prompt_token_ids, truncate_prompt_tokens) self.llm_engine.add_request(request_id, prompt, sampling_params, @@ -264,9 +265,13 @@ def _validate_prompt( truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None ) -> Tuple[Optional[str], Optional[List[int]]]: + if not (prompt or prompt_token_ids): + raise ValueError("Either prompt or prompt_ids should be provided.") + if (prompt and prompt_token_ids): + raise ValueError( + "Only one of prompt or prompt_ids should be provided.") if truncate_prompt_tokens is not None: if prompt_token_ids is None: - assert prompt is not None prompt_token_ids = self.llm_engine.tokenizer.tokenizer( prompt).input_ids prompt_token_ids = prompt_token_ids[-truncate_prompt_tokens:]