Merge branch 'main' into feat/lm-studio

src/libs/langchain/file.ts

@@ -10,6 +10,7 @@ export const LANGCHAIN_SUPPORT_TEXT_LIST = [
 
   'sh',
   'patch',
+  'log',
   // js
   'js',
   'jsx',

src/prompts/files/file.ts

@@ -0,0 +1,15 @@
+import { ChatFileItem } from '@/types/message';
+
+const filePrompt = (item: ChatFileItem) =>
+  `<file id="${item.id}" name="${item.name}" type="${item.fileType}" size="${item.size}" url="${item.url}"></file>`;
+
+export const filePrompts = (fileList: ChatFileItem[]) => {
+  if (fileList.length === 0) return '';
+
+  const prompt = `<files>
+<files_docstring>here are user upload files you can refer to</files_docstring>
+${fileList.map((item) => filePrompt(item)).join('\n')}
+</files>`;
+
+  return prompt.trim();
+};

src/prompts/files/image.ts

@@ -0,0 +1,14 @@
+import { ChatImageItem } from '@/types/message';
+
+const imagePrompt = (item: ChatImageItem) => `<image name="${item.alt}" url="${item.url}"></image>`;
+
+export const imagesPrompts = (imageList: ChatImageItem[]) => {
+  if (imageList.length === 0) return '';
+
+  const prompt = `<images>
+<images_docstring>here are user upload images you can refer to</images_docstring>
+${imageList.map((item) => imagePrompt(item)).join('\n')}
+</images>`;
+
+  return prompt.trim();
+};

src/prompts/files/index.ts

@@ -0,0 +1,19 @@
+import { ChatFileItem, ChatImageItem } from '@/types/message';
+
+import { filePrompts } from './file';
+import { imagesPrompts } from './image';
+
+export const filesPrompts = ({
+  imageList,
+  fileList,
+}: {
+  fileList?: ChatFileItem[];
+  imageList: ChatImageItem[];
+}) => {
+  const prompt = `<files_info>
+${imagesPrompts(imageList)}
+${fileList ? filePrompts(fileList) : ''}
+</files_info>`;
+
+  return prompt.trim();
+};

src/services/__tests__/chat.test.ts

@@ -126,6 +126,92 @@ describe('ChatService', () => {
       );
     });
 
+    describe('handle with files content', () => {
+      it('should includes files', async () => {
+        const messages = [
+          {
+            content: 'Hello',
+            role: 'user',
+            imageList: [
+              {
+                id: 'imagecx1',
+                url: 'http://example.com/xxx0asd-dsd.png',
+                alt: 'ttt.png',
+              },
+            ],
+            fileList: [
+              {
+                fileType: 'plain/txt',
+                size: 100000,
+                id: 'file1',
+                url: 'http://abc.com/abc.txt',
+                name: 'abc.png',
+              },
+              {
+                id: 'file_oKMve9qySLMI',
+                name: '2402.16667v1.pdf',
+                type: 'application/pdf',
+                size: 11256078,
+                url: 'https://xxx.com/ppp/480497/5826c2b8-fde0-4de1-a54b-a224d5e3d898.pdf',
+              },
+            ],
+          }, // Message with files
+          { content: 'Hi', role: 'tool', plugin: { identifier: 'plugin1', apiName: 'api1' } }, // Message with tool role
+          { content: 'Hey', role: 'assistant' }, // Regular user message
+        ] as ChatMessage[];
+
+        const getChatCompletionSpy = vi.spyOn(chatService, 'getChatCompletion');
+        await chatService.createAssistantMessage({
+          messages,
+          plugins: [],
+          model: 'gpt-4o',
+        });
+
+        expect(getChatCompletionSpy).toHaveBeenCalledWith(
+          {
+            messages: [
+              {
+                content: [
+                  {
+                    text: `Hello
+
+<files_info>
+<images>
+<images_docstring>here are user upload images you can refer to</images_docstring>
+<image name="ttt.png" url="http://example.com/xxx0asd-dsd.png"></image>
+</images>
+<files>
+<files_docstring>here are user upload files you can refer to</files_docstring>
+<file id="file1" name="abc.png" type="plain/txt" size="100000" url="http://abc.com/abc.txt"></file>
+<file id="file_oKMve9qySLMI" name="2402.16667v1.pdf" type="undefined" size="11256078" url="https://xxx.com/ppp/480497/5826c2b8-fde0-4de1-a54b-a224d5e3d898.pdf"></file>
+</files>
+</files_info>`,
+                    type: 'text',
+                  },
+                  {
+                    image_url: { detail: 'auto', url: 'http://example.com/xxx0asd-dsd.png' },
+                    type: 'image_url',
+                  },
+                ],
+                role: 'user',
+              },
+              {
+                content: 'Hi',
+                name: 'plugin1____api1',
+                role: 'tool',
+              },
+              {
+                content: 'Hey',
+                role: 'assistant',
+              },
+            ],
+            model: 'gpt-4o',
+          },
+          undefined,
+        );
+      });
+    });
+
     describe('should handle content correctly for vision models', () => {
       it('should include image content when with vision model', async () => {
         const messages = [
@@ -156,7 +242,18 @@ describe('ChatService', () => {
             messages: [
               {
                 content: [
-                  { text: 'Hello', type: 'text' },
+                  {
+                    text: `Hello
+
+<files_info>
+<images>
+<images_docstring>here are user upload images you can refer to</images_docstring>
+<image name="abc.png" url="http://example.com/image.jpg"></image>
+</images>
+
+</files_info>`,
+                    type: 'text',
+                  },
                   {
                     image_url: { detail: 'auto', url: 'http://example.com/image.jpg' },
                     type: 'image_url',

src/services/chat.ts

@@ -8,6 +8,7 @@ import { INBOX_SESSION_ID } from '@/const/session';
 import { DEFAULT_AGENT_CONFIG } from '@/const/settings';
 import { TracePayload, TraceTagMap } from '@/const/trace';
 import { AgentRuntime, ChatCompletionErrorPayload, ModelProvider } from '@/libs/agent-runtime';
+import { filesPrompts } from '@/prompts/files';
 import { useSessionStore } from '@/store/session';
 import { sessionMetaSelectors } from '@/store/session/selectors';
 import { useToolStore } from '@/store/tool';
@@ -413,22 +414,15 @@ class ChatService {
     // for the models with visual ability, add image url to content
     // refs: https://platform.openai.com/docs/guides/vision/quick-start
     const getContent = (m: ChatMessage) => {
-      if (!m.imageList) return m.content;
-
-      const imageList = m.imageList;
-
-      if (imageList.length === 0) return m.content;
-
-      const canUploadFile = modelProviderSelectors.isModelEnabledUpload(model)(
-        useUserStore.getState(),
-      );
-
-      if (!canUploadFile) {
+      // only if message doesn't have images and files, then return the plain content
+      if ((!m.imageList || m.imageList.length === 0) && (!m.fileList || m.fileList.length === 0))
         return m.content;
-      }
 
+      const imageList = m.imageList || [];
+
+      const filesContext = filesPrompts({ fileList: m.fileList, imageList });
       return [
-        { text: m.content, type: 'text' },
+        { text: m.content + '\n\n' + filesContext, type: 'text' },
         ...imageList.map(
           (i) => ({ image_url: { detail: 'auto', url: i.url }, type: 'image_url' }) as const,
         ),