Skip to content

Commit

Permalink
fix dependency injection
Browse files Browse the repository at this point in the history
  • Loading branch information
yecohn committed May 4, 2024
1 parent ae29377 commit 968d5e9
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions vllm/entrypoints/llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,8 +224,9 @@ def _add_request(
multi_modal_data: Optional[MultiModalData] = None,
) -> None:
request_id = str(next(self.request_counter))
truncate_prompt_tokens = sampling_params.truncate_prompt_tokens
prompt, prompt_token_ids = self._validate_prompt(
prompt, prompt_token_ids, sampling_params)
prompt, prompt_token_ids, truncate_prompt_tokens)
self.llm_engine.add_request(request_id,
prompt,
sampling_params,
Expand Down Expand Up @@ -264,9 +265,13 @@ def _validate_prompt(
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
) -> Tuple[Optional[str], Optional[List[int]]]:

if not (prompt or prompt_token_ids):
raise ValueError("Either prompt or prompt_ids should be provided.")
if (prompt and prompt_token_ids):
raise ValueError(
"Only one of prompt or prompt_ids should be provided.")
if truncate_prompt_tokens is not None:
if prompt_token_ids is None:
assert prompt is not None
prompt_token_ids = self.llm_engine.tokenizer.tokenizer(
prompt).input_ids
prompt_token_ids = prompt_token_ids[-truncate_prompt_tokens:]
Expand Down

0 comments on commit 968d5e9

Please sign in to comment.