Commit 84e8ebd 1 parent 940dc72 commit 84e8ebd Copy full SHA for 84e8ebd
File tree 1 file changed +5
-6
lines changed
1 file changed +5
-6
lines changed Original file line number Diff line number Diff line change @@ -265,6 +265,9 @@ async def async_request_openai_completions(
265
265
else :
266
266
data = json .loads (chunk )
267
267
268
+ # NOTE: Some completion API might have a last
269
+ # usage summary response without a token so we
270
+ # want to check a token was generated
268
271
if data ["choices" ][0 ]["text" ]:
269
272
timestamp = time .perf_counter ()
270
273
# First token
@@ -273,12 +276,8 @@ async def async_request_openai_completions(
273
276
output .ttft = ttft
274
277
275
278
# Decoding phase
276
- # NOTE: Some completion API might have a last
277
- # usage summary response without a token so we
278
- # do not want to include as inter-token-latency
279
- elif data .get ("usage" , None ) is None :
280
- output .itl .append (timestamp -
281
- most_recent_timestamp )
279
+ output .itl .append (timestamp -
280
+ most_recent_timestamp )
282
281
283
282
most_recent_timestamp = timestamp
284
283
generated_text += data ["choices" ][0 ]["text" ]
You can’t perform that action at this time.
0 commit comments