Skip to content

Commit

Permalink
Extract request's trace context in GenerateStream
Browse files Browse the repository at this point in the history
  • Loading branch information
ronensc authored and dtrifiro committed Aug 7, 2024
1 parent 09290e6 commit 3953bd8
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion src/vllm_tgis_adapter/grpc/grpc_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -345,14 +345,21 @@ async def GenerateStream(
prompt=request.request.text,
prompt_token_ids=input_ids,
)

kwargs = {}
is_tracing_enabled = await self.engine.is_tracing_enabled()
headers = dict(context.invocation_metadata())
if is_tracing_enabled:
kwargs["trace_headers"] = extract_trace_headers(headers)
elif contains_trace_headers(headers):
log_tracing_disabled_warning()
result_generator = self.engine.generate(
# prompt is supplied for observability, the text is not
# re-tokenized when `prompt_token_ids` is supplied
inputs=inputs,
sampling_params=sampling_params,
request_id=request_id,
**adapter_kwargs,
**kwargs,
)

resp_options = request.params.response
Expand Down

0 comments on commit 3953bd8

Please sign in to comment.