Fixing the oom maybe with 2.5.1 change. (#2958)

huggingface · Jan 28, 2025 · eb3df0f · eb3df0f
1 parent c690da5
commit eb3df0f
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py
@@ -28,7 +28,7 @@
     raise RuntimeError("Prefix caching is only supported with flashinfer")
 
 MEM_POOL = torch.cuda.graph_pool_handle() if torch.cuda.is_available() else None
-TGI_WIGGLE_ROOM = float(os.getenv("TGI_WIGGLE_ROOM", "0.95"))
+TGI_WIGGLE_ROOM = float(os.getenv("TGI_WIGGLE_ROOM", "0.93"))
 assert TGI_WIGGLE_ROOM > 0
 assert TGI_WIGGLE_ROOM < 1