From 79446d070a771e25cd8211776445e45858c4c3ad Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Wed, 11 Sep 2024 15:27:47 +0800 Subject: [PATCH] Add workflow rename; Fix: userselect chatId unrefresh (#2672) * feat: workflow node support rename * perf: push data to training queue * fix: userselect chatId unrefresh --- .../zh-cn/docs/development/upgrading/4811.md | 1 + .../core/dataset/training/controller.ts | 108 ++++++++-------- .../ChatBox/components/ChatController.tsx | 2 - .../ChatBox/components/ChatItem.tsx | 17 +-- .../core/chat/ChatContainer/ChatBox/index.tsx | 15 +-- .../core/chat/ChatContainer/useChat.ts | 5 +- .../core/chat/components/AIResponseBox.tsx | 115 +++++++----------- .../nodes/NodePluginIO/NodePluginConfig.tsx | 1 - .../Flow/nodes/NodePluginIO/PluginInput.tsx | 1 - .../Flow/nodes/NodePluginIO/PluginOutput.tsx | 1 - .../Flow/nodes/NodeSystemConfig.tsx | 1 - .../Flow/nodes/NodeWorkflowStart.tsx | 1 - .../Flow/nodes/render/NodeCard.tsx | 57 ++++----- projects/app/src/pages/chat/index.tsx | 7 +- 14 files changed, 144 insertions(+), 188 deletions(-) diff --git a/docSite/content/zh-cn/docs/development/upgrading/4811.md b/docSite/content/zh-cn/docs/development/upgrading/4811.md index 5fbcd0d7fd2..c2aac6ad15e 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4811.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4811.md @@ -25,3 +25,4 @@ weight: 813 8. 优化 - 工作流嵌套层级限制 20 层，避免因编排不合理导致的无限死循环。 9. 优化 - 工作流 handler 性能优化。 10. 修复 - 知识库选择权限问题。 +11. 修复 - 空 chatId 发起对话，首轮携带用户选择时会异常。 diff --git a/packages/service/core/dataset/training/controller.ts b/packages/service/core/dataset/training/controller.ts index 984e4a3c05c..e293a8183e7 100644 --- a/packages/service/core/dataset/training/controller.ts +++ b/packages/service/core/dataset/training/controller.ts @@ -10,6 +10,7 @@ import { ClientSession } from '../../../common/mongo'; import { getLLMModel, getVectorModel } from '../../ai/model'; import { addLog } from '../../../common/system/log'; import { getCollectionWithDataset } from '../controller'; +import { mongoSessionRun } from '../../../common/mongo/sessionRun'; export const lockTrainingDataByTeamId = async (teamId: string): Promise => { try { @@ -64,7 +65,7 @@ export async function pushDataListToTrainingQueue({ vectorModel: string; session?: ClientSession; } & PushDatasetDataProps): Promise { - const checkModelValid = async () => { + const { model, maxToken, weight } = await (async () => { const agentModelData = getLLMModel(agentModel); if (!agentModelData) { return Promise.reject(`File model ${agentModel} is inValid`); @@ -91,9 +92,16 @@ export async function pushDataListToTrainingQueue({ } return Promise.reject(`Training mode "${trainingMode}" is inValid`); - }; + })(); - const { model, maxToken, weight } = await checkModelValid(); + // filter repeat or equal content + const set = new Set(); + const filterResult: Record = { + success: [], + overToken: [], + repeat: [], + error: [] + }; // format q and a, remove empty char data.forEach((item) => { @@ -108,19 +116,8 @@ export async function pushDataListToTrainingQueue({ }; }) .filter(Boolean); - }); - - // filter repeat or equal content - const set = new Set(); - const filterResult: Record = { - success: [], - overToken: [], - repeat: [], - error: [] - }; - // filter repeat content - data.forEach((item) => { + // filter repeat content if (!item.q) { filterResult.error.push(item); return; @@ -150,40 +147,55 @@ export async function pushDataListToTrainingQueue({ const failedDocuments: PushDatasetDataChunkProps[] = []; // 使用 insertMany 批量插入 - try { - await MongoDatasetTraining.insertMany( - filterResult.success.map((item) => ({ - teamId, - tmbId, - datasetId, - collectionId, - billId, - mode: trainingMode, - prompt, - model, - q: item.q, - a: item.a, - chunkIndex: item.chunkIndex ?? 0, - weight: weight ?? 0, - indexes: item.indexes - })), - { - session, - ordered: false - } - ); - } catch (error: any) { - addLog.error(`Insert error`, error); - // 如果有错误，将失败的文档添加到失败列表中 - error.writeErrors?.forEach((writeError: any) => { - failedDocuments.push(data[writeError.index]); - }); - console.log('failed', failedDocuments); - } + const batchSize = 200; + const insertData = async (startIndex: number, session: ClientSession) => { + const list = filterResult.success.slice(startIndex, startIndex + batchSize); + + if (list.length === 0) return; + + try { + await MongoDatasetTraining.insertMany( + list.map((item) => ({ + teamId, + tmbId, + datasetId, + collectionId, + billId, + mode: trainingMode, + prompt, + model, + q: item.q, + a: item.a, + chunkIndex: item.chunkIndex ?? 0, + weight: weight ?? 0, + indexes: item.indexes + })), + { + session, + ordered: true + } + ); + } catch (error: any) { + addLog.error(`Insert error`, error); + // 如果有错误，将失败的文档添加到失败列表中 + error.writeErrors?.forEach((writeError: any) => { + failedDocuments.push(data[writeError.index]); + }); + console.log('failed', failedDocuments); + } + console.log(startIndex, '==='); + // 对于失败的文档，尝试单独插入 + await MongoDatasetTraining.create(failedDocuments, { session }); - // 对于失败的文档，尝试单独插入 - for await (const item of failedDocuments) { - await MongoDatasetTraining.create(item); + return insertData(startIndex + batchSize, session); + }; + + if (session) { + await insertData(0, session); + } else { + await mongoSessionRun(async (session) => { + await insertData(0, session); + }); } delete filterResult.success; diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatController.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatController.tsx index 368b2c10a0e..24c43c64bae 100644 --- a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatController.tsx +++ b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatController.tsx @@ -9,13 +9,11 @@ import { formatChatValue2InputType } from '../utils'; import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; import { ChatBoxContext } from '../Provider'; import { useContextSelector } from 'use-context-selector'; -import { SendPromptFnType } from '../type'; export type ChatControllerProps = { isLastChild: boolean; chat: ChatSiteItemType; showVoiceIcon?: boolean; - onSendMessage: SendPromptFnType; onRetry?: () => void; onDelete?: () => void; onMark?: () => void; diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatItem.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatItem.tsx index 3bc11ee14bb..e245a6eec07 100644 --- a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatItem.tsx +++ b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatItem.tsx @@ -19,7 +19,6 @@ import { useCopyData } from '@/web/common/hooks/useCopyData'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import { useTranslation } from 'next-i18next'; -import { SendPromptFnType } from '../type'; import { AIChatItemValueItemType, ChatItemValueItemType } from '@fastgpt/global/core/chat/type'; import { CodeClassNameEnum } from '@/components/Markdown/utils'; import { isEqual } from 'lodash'; @@ -51,7 +50,6 @@ type BasicProps = { type Props = BasicProps & { type: ChatRoleEnum.Human | ChatRoleEnum.AI; - onSendMessage: SendPromptFnType; }; const RenderQuestionGuide = ({ questionGuides }: { questionGuides: string[] }) => { @@ -80,14 +78,12 @@ const AIContentCard = React.memo(function AIContentCard({ dataId, isLastChild, isChatting, - onSendMessage, questionGuides }: { dataId: string; chatValue: ChatItemValueItemType[]; isLastChild: boolean; isChatting: boolean; - onSendMessage: SendPromptFnType; questionGuides: string[]; }) { return ( @@ -101,7 +97,6 @@ const AIContentCard = React.memo(function AIContentCard({ value={value} isLastChild={isLastChild && i === chatValue.length - 1} isChatting={isChatting} - onSendMessage={onSendMessage} /> ); })} @@ -113,16 +108,7 @@ const AIContentCard = React.memo(function AIContentCard({ }); const ChatItem = (props: Props) => { - const { - type, - avatar, - statusBoxData, - children, - isLastChild, - questionGuides = [], - onSendMessage, - chat - } = props; + const { type, avatar, statusBoxData, children, isLastChild, questionGuides = [], chat } = props; const styleMap: BoxProps = type === ChatRoleEnum.Human @@ -270,7 +256,6 @@ const ChatItem = (props: Props) => { dataId={chat.dataId} isLastChild={isLastChild && i === splitAiResponseResults.length - 1} isChatting={isChatting} - onSendMessage={onSendMessage} questionGuides={questionGuides} /> )} diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx index 873c9b5f952..e71931b9099 100644 --- a/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx +++ b/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx @@ -60,7 +60,7 @@ import dynamic from 'next/dynamic'; import type { StreamResponseType } from '@/web/common/api/fetch'; import { useContextSelector } from 'use-context-selector'; import { useSystem } from '@fastgpt/web/hooks/useSystem'; -import { useCreation, useMemoizedFn, useThrottleFn, useTrackedEffect } from 'ahooks'; +import { useCreation, useMemoizedFn, useThrottleFn } from 'ahooks'; import MyIcon from '@fastgpt/web/components/common/Icon'; const ResponseTags = dynamic(() => import('./components/ResponseTags')); @@ -832,12 +832,10 @@ const ChatBox = ( }; window.addEventListener('message', windowMessage); - eventBus.on(EventNameEnum.sendQuestion, ({ text }: { text: string }) => { - if (!text) return; - sendPrompt({ - text - }); - }); + const fn: SendPromptFnType = (e) => { + sendPrompt(e); + }; + eventBus.on(EventNameEnum.sendQuestion, fn); eventBus.on(EventNameEnum.editQuestion, ({ text }: { text: string }) => { if (!text) return; resetInputVal({ text }); @@ -881,7 +879,6 @@ const ChatBox = ( onRetry={retryInput(item.dataId)} onDelete={delOneMessage(item.dataId)} isLastChild={index === chatHistories.length - 1} - onSendMessage={sendPrompt} /> )} {item.obj === ChatRoleEnum.AI && ( @@ -891,7 +888,6 @@ const ChatBox = ( avatar={appAvatar} chat={item} isLastChild={index === chatHistories.length - 1} - onSendMessage={sendPrompt} {...{ showVoiceIcon, shareId, @@ -977,7 +973,6 @@ const ChatBox = ( outLinkUid, questionGuides, retryInput, - sendPrompt, shareId, showEmpty, showMarkIcon, diff --git a/projects/app/src/components/core/chat/ChatContainer/useChat.ts b/projects/app/src/components/core/chat/ChatContainer/useChat.ts index 124aebc5615..16f1590938d 100644 --- a/projects/app/src/components/core/chat/ChatContainer/useChat.ts +++ b/projects/app/src/components/core/chat/ChatContainer/useChat.ts @@ -2,7 +2,8 @@ import { ChatSiteItemType } from '@fastgpt/global/core/chat/type'; import { useCallback, useRef, useState } from 'react'; import { useForm } from 'react-hook-form'; import { PluginRunBoxTabEnum } from './PluginRunBox/constants'; -import { ComponentRef as ChatComponentRef } from './ChatBox/type'; +import { ComponentRef as ChatComponentRef, SendPromptFnType } from './ChatBox/type'; +import { eventBus, EventNameEnum } from '@/web/common/utils/eventbus'; export const useChat = () => { const ChatBoxRef = useRef(null); @@ -61,3 +62,5 @@ export const useChat = () => { resetChatRecords }; }; + +export const onSendPrompt: SendPromptFnType = (e) => eventBus.emit(EventNameEnum.sendQuestion, e); diff --git a/projects/app/src/components/core/chat/components/AIResponseBox.tsx b/projects/app/src/components/core/chat/components/AIResponseBox.tsx index f4a08ea67a6..c67dd9ab1e1 100644 --- a/projects/app/src/components/core/chat/components/AIResponseBox.tsx +++ b/projects/app/src/components/core/chat/components/AIResponseBox.tsx @@ -12,24 +12,20 @@ import { import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants'; import { AIChatItemValueItemType, - ChatSiteItemType, ToolModuleResponseItemType, UserChatItemValueItemType } from '@fastgpt/global/core/chat/type'; import React from 'react'; import MyIcon from '@fastgpt/web/components/common/Icon'; import Avatar from '@fastgpt/web/components/common/Avatar'; -import { SendPromptFnType } from '../ChatContainer/ChatBox/type'; -import { useContextSelector } from 'use-context-selector'; -import { ChatBoxContext } from '../ChatContainer/ChatBox/Provider'; import { InteractiveNodeResponseItemType } from '@fastgpt/global/core/workflow/template/system/userSelect/type'; import { isEqual } from 'lodash'; +import { onSendPrompt } from '../ChatContainer/useChat'; type props = { value: UserChatItemValueItemType | AIChatItemValueItemType; isLastChild: boolean; isChatting: boolean; - onSendMessage?: SendPromptFnType; }; const RenderText = React.memo(function RenderText({ @@ -128,67 +124,51 @@ ${toolResponse}`} }, (prevProps, nextProps) => isEqual(prevProps, nextProps) ); -const RenderInteractive = React.memo( - function RenderInteractive({ - isChatting, - interactive, - onSendMessage, - chatHistories - }: { - isChatting: boolean; - interactive: InteractiveNodeResponseItemType; - onSendMessage?: SendPromptFnType; - chatHistories: ChatSiteItemType[]; - }) { - return ( - <> - {interactive?.params?.description && } - - {interactive.params.userSelectOptions?.map((option) => { - const selected = option.value === interactive?.params?.userSelectedVal; +const RenderInteractive = React.memo(function RenderInteractive({ + interactive +}: { + interactive: InteractiveNodeResponseItemType; +}) { + return ( + <> + {interactive?.params?.description && } + + {interactive.params.userSelectOptions?.map((option) => { + const selected = option.value === interactive?.params?.userSelectedVal; - return ( - - ); - })} - - - ); - }, - ( - prevProps, - nextProps // isChatting 更新时候，onSendMessage 和 chatHistories 肯定都更新了，这里不需要额外的刷新 - ) => - prevProps.isChatting === nextProps.isChatting && - isEqual(prevProps.interactive, nextProps.interactive) -); - -const AIResponseBox = ({ value, isLastChild, isChatting, onSendMessage }: props) => { - const chatHistories = useContextSelector(ChatBoxContext, (v) => v.chatHistories); + } + : {})} + onClick={() => { + onSendPrompt({ + text: option.value, + isInteractivePrompt: true + }); + }} + > + {option.value} + + ); + })} + + + ); +}); +const AIResponseBox = ({ value, isLastChild, isChatting }: props) => { if (value.type === ChatItemValueTypeEnum.text && value.text) return ; if (value.type === ChatItemValueTypeEnum.tool && value.tools) @@ -198,14 +178,7 @@ const AIResponseBox = ({ value, isLastChild, isChatting, onSendMessage }: props) value.interactive && value.interactive.type === 'userSelect' ) - return ( - - ); + return ; }; export default React.memo(AIResponseBox); diff --git a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/NodePluginConfig.tsx b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/NodePluginConfig.tsx index e539467d45e..de815750a18 100644 --- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/NodePluginConfig.tsx +++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/NodePluginConfig.tsx @@ -53,7 +53,6 @@ const NodePluginConfig = ({ data, selected }: NodeProps) => { selected={selected} menuForbid={{ debug: true, - rename: true, copy: true, delete: true }} diff --git a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/PluginInput.tsx b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/PluginInput.tsx index 535e30f3b61..af9a312d833 100644 --- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/PluginInput.tsx +++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/PluginInput.tsx @@ -91,7 +91,6 @@ const NodePluginInput = ({ data, selected }: NodeProps) => { minW={'300px'} selected={selected} menuForbid={{ - rename: true, copy: true, delete: true }} diff --git a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/PluginOutput.tsx b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/PluginOutput.tsx index 9e7580eadf1..548e0bc306e 100644 --- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/PluginOutput.tsx +++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodePluginIO/PluginOutput.tsx @@ -48,7 +48,6 @@ const NodePluginOutput = ({ data, selected }: NodeProps) => { selected={selected} menuForbid={{ debug: true, - rename: true, copy: true, delete: true }} diff --git a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodeSystemConfig.tsx b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodeSystemConfig.tsx index e96389990d1..331ecccde47 100644 --- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodeSystemConfig.tsx +++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodeSystemConfig.tsx @@ -53,7 +53,6 @@ const NodeUserGuide = ({ data, selected }: NodeProps) => { selected={selected} menuForbid={{ debug: true, - rename: true, copy: true, delete: true }} diff --git a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodeWorkflowStart.tsx b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodeWorkflowStart.tsx index ea71243c5b6..0f9a77c972e 100644 --- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodeWorkflowStart.tsx +++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/NodeWorkflowStart.tsx @@ -68,7 +68,6 @@ const NodeStart = ({ data, selected }: NodeProps) => { minW={'240px'} selected={selected} menuForbid={{ - rename: true, copy: true, delete: true }} diff --git a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/NodeCard.tsx b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/NodeCard.tsx index ea1ee324a91..f74d7a28ad9 100644 --- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/NodeCard.tsx +++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/NodeCard.tsx @@ -33,7 +33,6 @@ type Props = FlowNodeItemType & { selected?: boolean; menuForbid?: { debug?: boolean; - rename?: boolean; copy?: boolean; delete?: boolean; }; @@ -154,37 +153,35 @@ const NodeCard = (props: Props) => { {t(name as any)} - {!menuForbid?.rename && ( - { - onOpenCustomTitleModal({ - defaultVal: name, - onSuccess: (e) => { - if (!e) { - return toast({ - title: t('app:modules.Title is required'), - status: 'warning' - }); - } - onChangeNode({ - nodeId, - type: 'attr', - key: 'name', - value: e + { + onOpenCustomTitleModal({ + defaultVal: name, + onSuccess: (e) => { + if (!e) { + return toast({ + title: t('app:modules.Title is required'), + status: 'warning' }); } - }); - }} - /> - )} + onChangeNode({ + nodeId, + type: 'attr', + key: 'name', + value: e + }); + } + }); + }} + /> {hasNewVersion && ( diff --git a/projects/app/src/pages/chat/index.tsx b/projects/app/src/pages/chat/index.tsx index c43317d1a9b..bf51a6c2d2d 100644 --- a/projects/app/src/pages/chat/index.tsx +++ b/projects/app/src/pages/chat/index.tsx @@ -130,7 +130,6 @@ const Chat = ({ const completionChatId = chatId || getNanoid(); // Just send a user prompt const histories = messages.slice(-1); - const { responseText, responseData } = await streamFetch({ data: { messages: histories, @@ -146,10 +145,8 @@ const Chat = ({ const newTitle = getChatTitleFromChatMessage(GPTMessages2Chats(histories)[0]); // new chat - if (completionChatId !== chatId) { - if (controller.signal.reason !== 'leave') { - onChangeChatId(completionChatId, true); - } + if (completionChatId !== chatId && controller.signal.reason !== 'leave') { + onChangeChatId(completionChatId, true); } loadHistories();