Add workflow rename; Fix: userselect chatId unrefresh (#2672)

* feat: workflow node support rename * perf: push data to training queue * fix: userselect chatId unrefresh
labring · Sep 11, 2024 · 02bf400 · 02bf400
1 parent 11cbcca
commit 02bf400
Show file tree

Hide file tree

Showing 14 changed files with 144 additions and 188 deletions.
diff --git a/docSite/content/zh-cn/docs/development/upgrading/4811.md b/docSite/content/zh-cn/docs/development/upgrading/4811.md
@@ -25,3 +25,4 @@ weight: 813
 8. 优化 - 工作流嵌套层级限制 20 层，避免因编排不合理导致的无限死循环。
 9. 优化 - 工作流 handler 性能优化。
 10. 修复 - 知识库选择权限问题。
+11. 修复 - 空 chatId 发起对话，首轮携带用户选择时会异常。
diff --git a/packages/service/core/dataset/training/controller.ts b/packages/service/core/dataset/training/controller.ts
@@ -10,6 +10,7 @@ import { ClientSession } from '../../../common/mongo';
 import { getLLMModel, getVectorModel } from '../../ai/model';
 import { addLog } from '../../../common/system/log';
 import { getCollectionWithDataset } from '../controller';
+import { mongoSessionRun } from '../../../common/mongo/sessionRun';
 
 export const lockTrainingDataByTeamId = async (teamId: string): Promise<any> => {
   try {
@@ -64,7 +65,7 @@ export async function pushDataListToTrainingQueue({
   vectorModel: string;
   session?: ClientSession;
 } & PushDatasetDataProps): Promise<PushDatasetDataResponse> {
-  const checkModelValid = async () => {
+  const { model, maxToken, weight } = await (async () => {
     const agentModelData = getLLMModel(agentModel);
     if (!agentModelData) {
       return Promise.reject(`File model ${agentModel} is inValid`);
@@ -91,9 +92,16 @@ export async function pushDataListToTrainingQueue({
     }
 
     return Promise.reject(`Training mode "${trainingMode}" is inValid`);
-  };
+  })();
 
-  const { model, maxToken, weight } = await checkModelValid();
+  // filter repeat or equal content
+  const set = new Set();
+  const filterResult: Record<string, PushDatasetDataChunkProps[]> = {
+    success: [],
+    overToken: [],
+    repeat: [],
+    error: []
+  };
 
   // format q and a, remove empty char
   data.forEach((item) => {
@@ -108,19 +116,8 @@ export async function pushDataListToTrainingQueue({
         };
       })
       .filter(Boolean);
-  });
-
-  // filter repeat or equal content
-  const set = new Set();
-  const filterResult: Record<string, PushDatasetDataChunkProps[]> = {
-    success: [],
-    overToken: [],
-    repeat: [],
-    error: []
-  };
 
-  // filter repeat content
-  data.forEach((item) => {
+    // filter repeat content
     if (!item.q) {
       filterResult.error.push(item);
       return;
@@ -150,40 +147,55 @@ export async function pushDataListToTrainingQueue({
   const failedDocuments: PushDatasetDataChunkProps[] = [];
 
   // 使用 insertMany 批量插入
-  try {
-    await MongoDatasetTraining.insertMany(
-      filterResult.success.map((item) => ({
-        teamId,
-        tmbId,
-        datasetId,
-        collectionId,
-        billId,
-        mode: trainingMode,
-        prompt,
-        model,
-        q: item.q,
-        a: item.a,
-        chunkIndex: item.chunkIndex ?? 0,
-        weight: weight ?? 0,
-        indexes: item.indexes
-      })),
-      {
-        session,
-        ordered: false
-      }
-    );
-  } catch (error: any) {
-    addLog.error(`Insert error`, error);
-    // 如果有错误，将失败的文档添加到失败列表中
-    error.writeErrors?.forEach((writeError: any) => {
-      failedDocuments.push(data[writeError.index]);
-    });
-    console.log('failed', failedDocuments);
-  }
+  const batchSize = 200;
+  const insertData = async (startIndex: number, session: ClientSession) => {
+    const list = filterResult.success.slice(startIndex, startIndex + batchSize);
+
+    if (list.length === 0) return;
+
+    try {
+      await MongoDatasetTraining.insertMany(
+        list.map((item) => ({
+          teamId,
+          tmbId,
+          datasetId,
+          collectionId,
+          billId,
+          mode: trainingMode,
+          prompt,
+          model,
+          q: item.q,
+          a: item.a,
+          chunkIndex: item.chunkIndex ?? 0,
+          weight: weight ?? 0,
+          indexes: item.indexes
+        })),
+        {
+          session,
+          ordered: true
+        }
+      );
+    } catch (error: any) {
+      addLog.error(`Insert error`, error);
+      // 如果有错误，将失败的文档添加到失败列表中
+      error.writeErrors?.forEach((writeError: any) => {
+        failedDocuments.push(data[writeError.index]);
+      });
+      console.log('failed', failedDocuments);
+    }
+    console.log(startIndex, '===');
+    // 对于失败的文档，尝试单独插入
+    await MongoDatasetTraining.create(failedDocuments, { session });
 
-  // 对于失败的文档，尝试单独插入
-  for await (const item of failedDocuments) {
-    await MongoDatasetTraining.create(item);
+    return insertData(startIndex + batchSize, session);
+  };
+
+  if (session) {
+    await insertData(0, session);
+  } else {
+    await mongoSessionRun(async (session) => {
+      await insertData(0, session);
+    });
   }
 
   delete filterResult.success;

diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatController.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatController.tsx
@@ -9,13 +9,11 @@ import { formatChatValue2InputType } from '../utils';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import { ChatBoxContext } from '../Provider';
 import { useContextSelector } from 'use-context-selector';
-import { SendPromptFnType } from '../type';
 
 export type ChatControllerProps = {
   isLastChild: boolean;
   chat: ChatSiteItemType;
   showVoiceIcon?: boolean;
-  onSendMessage: SendPromptFnType;
   onRetry?: () => void;
   onDelete?: () => void;
   onMark?: () => void;

diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatItem.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatItem.tsx
@@ -19,7 +19,6 @@ import { useCopyData } from '@/web/common/hooks/useCopyData';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 import { useTranslation } from 'next-i18next';
-import { SendPromptFnType } from '../type';
 import { AIChatItemValueItemType, ChatItemValueItemType } from '@fastgpt/global/core/chat/type';
 import { CodeClassNameEnum } from '@/components/Markdown/utils';
 import { isEqual } from 'lodash';
@@ -51,7 +50,6 @@ type BasicProps = {
 
 type Props = BasicProps & {
   type: ChatRoleEnum.Human | ChatRoleEnum.AI;
-  onSendMessage: SendPromptFnType;
 };
 
 const RenderQuestionGuide = ({ questionGuides }: { questionGuides: string[] }) => {
@@ -80,14 +78,12 @@ const AIContentCard = React.memo(function AIContentCard({
   dataId,
   isLastChild,
   isChatting,
-  onSendMessage,
   questionGuides
 }: {
   dataId: string;
   chatValue: ChatItemValueItemType[];
   isLastChild: boolean;
   isChatting: boolean;
-  onSendMessage: SendPromptFnType;
   questionGuides: string[];
 }) {
   return (
@@ -101,7 +97,6 @@ const AIContentCard = React.memo(function AIContentCard({
             value={value}
             isLastChild={isLastChild && i === chatValue.length - 1}
             isChatting={isChatting}
-            onSendMessage={onSendMessage}
           />
         );
       })}
@@ -113,16 +108,7 @@ const AIContentCard = React.memo(function AIContentCard({
 });
 
 const ChatItem = (props: Props) => {
-  const {
-    type,
-    avatar,
-    statusBoxData,
-    children,
-    isLastChild,
-    questionGuides = [],
-    onSendMessage,
-    chat
-  } = props;
+  const { type, avatar, statusBoxData, children, isLastChild, questionGuides = [], chat } = props;
 
   const styleMap: BoxProps =
     type === ChatRoleEnum.Human
@@ -270,7 +256,6 @@ const ChatItem = (props: Props) => {
                 dataId={chat.dataId}
                 isLastChild={isLastChild && i === splitAiResponseResults.length - 1}
                 isChatting={isChatting}
-                onSendMessage={onSendMessage}
                 questionGuides={questionGuides}
               />
             )}

diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx
@@ -60,7 +60,7 @@ import dynamic from 'next/dynamic';
 import type { StreamResponseType } from '@/web/common/api/fetch';
 import { useContextSelector } from 'use-context-selector';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
-import { useCreation, useMemoizedFn, useThrottleFn, useTrackedEffect } from 'ahooks';
+import { useCreation, useMemoizedFn, useThrottleFn } from 'ahooks';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 
 const ResponseTags = dynamic(() => import('./components/ResponseTags'));
@@ -832,12 +832,10 @@ const ChatBox = (
     };
     window.addEventListener('message', windowMessage);
 
-    eventBus.on(EventNameEnum.sendQuestion, ({ text }: { text: string }) => {
-      if (!text) return;
-      sendPrompt({
-        text
-      });
-    });
+    const fn: SendPromptFnType = (e) => {
+      sendPrompt(e);
+    };
+    eventBus.on(EventNameEnum.sendQuestion, fn);
     eventBus.on(EventNameEnum.editQuestion, ({ text }: { text: string }) => {
       if (!text) return;
       resetInputVal({ text });
@@ -881,7 +879,6 @@ const ChatBox = (
                     onRetry={retryInput(item.dataId)}
                     onDelete={delOneMessage(item.dataId)}
                     isLastChild={index === chatHistories.length - 1}
-                    onSendMessage={sendPrompt}
                   />
                 )}
                 {item.obj === ChatRoleEnum.AI && (
@@ -891,7 +888,6 @@ const ChatBox = (
                       avatar={appAvatar}
                       chat={item}
                       isLastChild={index === chatHistories.length - 1}
-                      onSendMessage={sendPrompt}
                       {...{
                         showVoiceIcon,
                         shareId,
@@ -977,7 +973,6 @@ const ChatBox = (
     outLinkUid,
     questionGuides,
     retryInput,
-    sendPrompt,
     shareId,
     showEmpty,
     showMarkIcon,

diff --git a/projects/app/src/components/core/chat/ChatContainer/useChat.ts b/projects/app/src/components/core/chat/ChatContainer/useChat.ts
@@ -2,7 +2,8 @@ import { ChatSiteItemType } from '@fastgpt/global/core/chat/type';
 import { useCallback, useRef, useState } from 'react';
 import { useForm } from 'react-hook-form';
 import { PluginRunBoxTabEnum } from './PluginRunBox/constants';
-import { ComponentRef as ChatComponentRef } from './ChatBox/type';
+import { ComponentRef as ChatComponentRef, SendPromptFnType } from './ChatBox/type';
+import { eventBus, EventNameEnum } from '@/web/common/utils/eventbus';
 
 export const useChat = () => {
   const ChatBoxRef = useRef<ChatComponentRef>(null);
@@ -61,3 +62,5 @@ export const useChat = () => {
     resetChatRecords
   };
 };
+
+export const onSendPrompt: SendPromptFnType = (e) => eventBus.emit(EventNameEnum.sendQuestion, e);