[Bugfix] Miscalculated latency lead to time_to_first_token_seconds in…

…accurate. (vllm-project#6686)
KuntaiDu · Jul 24, 2024 · 787758a · 787758a
1 parent db593ae
commit 787758a
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py
@@ -949,8 +949,9 @@ def do_log_stats(
             model_output: Optional[List[SamplerOutput]] = None) -> None:
         """Forced log when no requests active."""
         if self.log_stats:
+            stats = self._get_stats(scheduler_outputs, model_output)
             for logger in self.stat_loggers.values():
-                logger.log(self._get_stats(scheduler_outputs, model_output))
+                logger.log(stats)
 
     def _get_stats(
             self,

diff --git a/vllm/spec_decode/spec_decode_worker.py b/vllm/spec_decode/spec_decode_worker.py
@@ -484,7 +484,7 @@ def _run_non_driver_rank(self) -> bool:
         for both speculation cases (num_lookahead_slots>0) and non-speculation
         cases (e.g. prefill).
 
-        Returns True iff there are remaining sequences to process.
+        Returns True if there are remaining sequences to process.
         """
         assert self.rank != self._driver_rank