Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat:add think tag display for xinference deepseek r1 #13291

Merged
merged 1 commit into from
Feb 6, 2025
+16 −6
Merged

feat:add think tag display for xinference deepseek r1 #13291

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions api/core/model_runtime/model_providers/xinference/llm/llm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import re
from collections.abc import Generator, Iterator
from typing import Optional, cast

Expand Down Expand Up @@ -635,16 +636,16 @@ def _handle_chat_stream_response(
handle stream chat generate response
"""
full_response = ""

is_reasoning_started_tag = False
for chunk in resp:
if len(chunk.choices) == 0:
continue

delta = chunk.choices[0]

if delta.finish_reason is None and (delta.delta.content is None or delta.delta.content == ""):
continue

delta_content = delta.delta.content
if not delta_content:
delta_content = ""
# check if there is a tool call in the response
function_call = None
tool_calls = []
Expand All @@ -657,9 +658,18 @@ def _handle_chat_stream_response(
if function_call:
assistant_message_tool_calls += [self._extract_response_function_call(function_call)]

if not is_reasoning_started_tag and "<think>" in delta_content:
is_reasoning_started_tag = True
delta_content = "> 💭 " + delta_content.replace("<think>", "")
elif is_reasoning_started_tag and "</think>" in delta_content:
delta_content = delta_content.replace("</think>", "") + "\n\n"
is_reasoning_started_tag = False
elif is_reasoning_started_tag:
if "\n" in delta_content:
delta_content = re.sub(r"\n(?!(>|\n))", "\n> ", delta_content)
# transform assistant message to prompt message
assistant_prompt_message = AssistantPromptMessage(
content=delta.delta.content or "", tool_calls=assistant_message_tool_calls
content=delta_content or "", tool_calls=assistant_message_tool_calls
)

if delta.finish_reason is not None:
Expand Down Expand Up @@ -697,7 +707,7 @@ def _handle_chat_stream_response(
),
)

full_response += delta.delta.content
full_response += delta_content

def _handle_completion_generate_response(
self,
Expand Down