From 173d2fa42eeb2412cd40608902a8216c0c273847 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 9 Jul 2024 02:23:24 +0800 Subject: [PATCH] [Bugfix] use diskcache in outlines _get_guide #5436 (#6203) --- .../guided_decoding/outlines_logits_processors.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/guided_decoding/outlines_logits_processors.py b/vllm/model_executor/guided_decoding/outlines_logits_processors.py index 1618705ff2983..1c8f6cccb3e9a 100644 --- a/vllm/model_executor/guided_decoding/outlines_logits_processors.py +++ b/vllm/model_executor/guided_decoding/outlines_logits_processors.py @@ -21,6 +21,7 @@ from typing import Callable, DefaultDict, Dict, List, Union import torch +from outlines.caching import cache from outlines.fsm.guide import CFGGuide, Generate, Guide, RegexGuide, Write from outlines.fsm.json_schema import build_regex_from_schema from pydantic import BaseModel @@ -67,7 +68,7 @@ def __call__(self, input_ids: List[int], class RegexLogitsProcessor(BaseLogitsProcessor): @classmethod - @lru_cache(maxsize=32) + @cache() def _get_guide(cls, regex_string: str, tokenizer: PreTrainedTokenizerBase) -> Guide: tokenizer = _adapt_tokenizer(tokenizer) @@ -126,7 +127,7 @@ def __init__(self, schema: Union[str, Dict, BaseModel], class CFGLogitsProcessor(BaseLogitsProcessor): @classmethod - @lru_cache(maxsize=32) + @cache() def _get_guide(cls, cfg: str, tokenizer: PreTrainedTokenizerBase) -> Guide: tokenizer = _adapt_tokenizer(tokenizer) return CFGGuide(cfg, tokenizer)