fix: fix stream options param drop

chore: In image dialogue, exclude the tool and tool call data from the chat history to avoid some models do not support null content.
adolphnov · Dec 5, 2024 · 3085c46 · 3085c46
1 parent 29ab2dc
commit 3085c46
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 10 deletions.
diff --git a/src/agent/model_middleware.ts b/src/agent/model_middleware.ts
@@ -3,6 +3,7 @@ import type {
     LanguageModelV1,
     LanguageModelV1CallOptions,
     Experimental_LanguageModelV1Middleware as LanguageModelV1Middleware,
+    LanguageModelV1Prompt,
     StepResult,
 } from 'ai';
 import type { ToolChoice } from '.';
@@ -113,15 +114,30 @@ export function AIMiddleware({ config, activeTools, onStream, toolChoice, messag
 
 function warpMessages(params: LanguageModelV1CallOptions, tools: Record<string, any>, activeTools: string[], rawSystemPrompt: string | undefined) {
     const { prompt: messages, mode } = params;
-
-    if (messages.at(-1)?.role === 'tool') {
-        const content = messages.at(-1)!.content;
-        if (Array.isArray(content) && content.length > 0) {
-            content.forEach((i: any) => {
-                delete i.result.time;
-            });
+    const trimMessages = (messages: LanguageModelV1Prompt) => {
+        if (messages.at(-1)?.role === 'tool') {
+            const content = messages.at(-1)!.content;
+            if (Array.isArray(content) && content.length > 0) {
+                content.forEach((i: any) => {
+                    delete i.result.time;
+                });
+            }
+        } else if (messages.at(-1)?.role === 'user') {
+            const content = messages.at(-1)!.content;
+            if (Array.isArray(content) && content.some(i => i.type === 'image')) {
+                const newMessages: LanguageModelV1Prompt = [];
+                for (const message of messages) {
+                    if (message.role === 'tool' || (message.role === 'assistant' && message.content.some(i => i.type === 'tool-call'))) {
+                        continue;
+                    }
+                    newMessages.push(message);
+                }
+                activeTools.length = 0;
+                params.prompt = newMessages;
+            }
         }
-    }
+    };
+    trimMessages(messages);
     if (activeTools.length > 0) {
         if (messages[0].role === 'system') {
             messages[0].content = `${rawSystemPrompt}\n\nYou can consider using the following tools:\n##TOOLS${activeTools.map(name =>

diff --git a/src/agent/openai.ts b/src/agent/openai.ts
@@ -59,7 +59,7 @@ export class OpenAI extends OpenAIBase implements ChatAgent {
         if (Object.keys(context.DROPS_OPENAI_PARAMS).length > 0) {
             for (const [model_perfix, params] of Object.entries(context.DROPS_OPENAI_PARAMS)) {
                 if (model_perfix.split(',').some(p => model.startsWith(p))) {
-                    params.includes('stream') && (onStream = null);
+                    params.split(',').includes('stream') && (onStream = null);
                     break;
                 }
             }

diff --git a/src/config/config.ts b/src/config/config.ts
@@ -397,7 +397,7 @@ export class ExtraUserConfig {
     // Jina Rerank Model
     JINA_RERANK_MODEL = 'jina-colbert-v2';
     // Rerank Models
-    RERANK_MODELS: string[] = ['gpt-4o-mini', 'gpt-4o-2024-05-13', 'gpt-4o-2024-08-06', 'chatgpt-4o-latest', 'o1-mini', 'o1-preview', 'claude-3-5-sonnet-20240620', 'claude-3-5-sonnet-20241012', 'gemini-1.5-flash-002', 'gemini-1.5-pro-002', 'gemini-1.5-flash-latest', 'gemini-1.5-pro-latest', 'gemini-exp-1114', 'grok-beta', 'grok-vision-beta', 'claude-3-5-haiku-20241012'];
+    RERANK_MODELS: string[] = ['gpt-4o-mini', 'gpt-4o-2024-05-13', 'gpt-4o-2024-08-06', 'chatgpt-4o-latest', 'o1-mini', 'o1-preview', 'claude-3-5-sonnet-20240620', 'claude-3-5-sonnet-20241012', 'gemini-1.5-flash-002', 'gemini-1.5-pro-002', 'gemini-1.5-flash-latest', 'gemini-1.5-pro-latest', 'gemini-exp-1121', 'grok-beta', 'grok-vision-beta', 'claude-3-5-haiku-20241012'];
     // Whether to enable intelligent model processing
     ENABLE_INTELLIGENT_MODEL = false;
     // text handle type, to 'tts' or 'text' to chat with llm, or 'chat' by using audio-preview (default: text)