From f4ba56f47222fc4395c0e1a6e061348937778ef0 Mon Sep 17 00:00:00 2001 From: bill Date: Mon, 2 Sep 2024 19:29:32 +0800 Subject: [PATCH] feat: Play audio #2088 --- web/package-lock.json | 6 ++ web/package.json | 1 + .../components/message-item/group-button.tsx | 9 +- web/src/components/message-item/hooks.ts | 54 +++++++++++- web/src/hooks/logic-hooks.ts | 82 +++++++++++++++++++ 5 files changed, 147 insertions(+), 5 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index 510a8c46883..28dd39f178d 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -28,6 +28,7 @@ "jsencrypt": "^3.3.2", "lodash": "^4.17.21", "mammoth": "^1.7.2", + "openai-speech-stream-player": "^1.0.8", "rc-tween-one": "^3.0.6", "react-copy-to-clipboard": "^5.1.0", "react-force-graph": "^1.44.4", @@ -20565,6 +20566,11 @@ "node": ">=12" } }, + "node_modules/openai-speech-stream-player": { + "version": "1.0.8", + "resolved": "https://registry.npmmirror.com/openai-speech-stream-player/-/openai-speech-stream-player-1.0.8.tgz", + "integrity": "sha512-0SUybbhStl65s66ezh2QaoZE5k1kNb2t5M8tDOqJFILdHpwHaBqnYy4uHl3Hk/8F5VFWxxHaLamjKOnfNDKgbw==" + }, "node_modules/option": { "version": "0.2.4", "resolved": "https://registry.npmmirror.com/option/-/option-0.2.4.tgz", diff --git a/web/package.json b/web/package.json index eae40b54824..326b2f7b1ae 100644 --- a/web/package.json +++ b/web/package.json @@ -39,6 +39,7 @@ "jsencrypt": "^3.3.2", "lodash": "^4.17.21", "mammoth": "^1.7.2", + "openai-speech-stream-player": "^1.0.8", "rc-tween-one": "^3.0.6", "react-copy-to-clipboard": "^5.1.0", "react-force-graph": "^1.44.4", diff --git a/web/src/components/message-item/group-button.tsx b/web/src/components/message-item/group-button.tsx index 6b433bafa71..cbbb8abc27b 100644 --- a/web/src/components/message-item/group-button.tsx +++ b/web/src/components/message-item/group-button.tsx @@ -5,6 +5,7 @@ import { DeleteOutlined, DislikeOutlined, LikeOutlined, + PauseCircleOutlined, SoundOutlined, SyncOutlined, } from '@ant-design/icons'; @@ -13,7 +14,7 @@ import { useCallback } from 'react'; import { useTranslation } from 'react-i18next'; import SvgIcon from '../svg-icon'; import FeedbackModal from './feedback-modal'; -import { useRemoveMessage, useSendFeedback } from './hooks'; +import { useRemoveMessage, useSendFeedback, useSpeech } from './hooks'; import PromptModal from './prompt-modal'; interface IProps { @@ -37,6 +38,7 @@ export const AssistantGroupButton = ({ showModal: showPromptModal, } = useSetModalState(); const { t } = useTranslation(); + const { handleRead, ref, isPlaying } = useSpeech(content); const handleLike = useCallback(() => { onFeedbackOk({ thumbup: true }); @@ -48,10 +50,11 @@ export const AssistantGroupButton = ({ - + - + {isPlaying ? : } + {showLikeButton && ( <> diff --git a/web/src/components/message-item/hooks.ts b/web/src/components/message-item/hooks.ts index 9e9a70f5cb7..1142509f4a3 100644 --- a/web/src/components/message-item/hooks.ts +++ b/web/src/components/message-item/hooks.ts @@ -1,9 +1,10 @@ import { useDeleteMessage, useFeedback } from '@/hooks/chat-hooks'; import { useSetModalState } from '@/hooks/common-hooks'; -import { IRemoveMessageById } from '@/hooks/logic-hooks'; +import { IRemoveMessageById, useSpeechWithSse } from '@/hooks/logic-hooks'; import { IFeedbackRequestBody } from '@/interfaces/request/chat'; import { getMessagePureId } from '@/utils/chat'; -import { useCallback } from 'react'; +import { SpeechPlayer } from 'openai-speech-stream-player'; +import { useCallback, useEffect, useRef, useState } from 'react'; export const useSendFeedback = (messageId: string) => { const { visible, hideModal, showModal } = useSetModalState(); @@ -50,3 +51,52 @@ export const useRemoveMessage = ( return { onRemoveMessage, loading }; }; + +export const useSpeech = (content: string) => { + const ref = useRef(null); + const { read } = useSpeechWithSse(); + const player = useRef(); + const [isPlaying, setIsPlaying] = useState(false); + + const initialize = useCallback(async () => { + player.current = new SpeechPlayer({ + audio: ref.current!, + onPlaying: () => { + setIsPlaying(true); + }, + onPause: () => { + setIsPlaying(false); + }, + onChunkEnd: () => {}, + mimeType: 'audio/mpeg', + }); + await player.current.init(); + }, []); + + const pause = useCallback(() => { + player.current?.pause(); + }, []); + + const speech = useCallback(async () => { + const response = await read({ text: content }); + if (response) { + player?.current?.feedWithResponse(response); + } + }, [read, content]); + + const handleRead = useCallback(async () => { + if (isPlaying) { + setIsPlaying(false); + pause(); + } else { + setIsPlaying(true); + speech(); + } + }, [setIsPlaying, speech, isPlaying, pause]); + + useEffect(() => { + initialize(); + }, [initialize]); + + return { ref, handleRead, isPlaying }; +}; diff --git a/web/src/hooks/logic-hooks.ts b/web/src/hooks/logic-hooks.ts index 6a283855907..80adf5188bb 100644 --- a/web/src/hooks/logic-hooks.ts +++ b/web/src/hooks/logic-hooks.ts @@ -278,6 +278,88 @@ export const useSendMessageWithSse = ( return { send, answer, done, setDone }; }; +export const useSpeechWithSse = (url: string = api.tts) => { + const read = useCallback( + (body: any) => { + const response = fetch(url, { + method: 'POST', + headers: { + [Authorization]: getAuthorization(), + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }); + return response; + }, + [url], + ); + + return { read }; +}; + +export const useFetchAudioWithSse = (url: string = api.tts) => { + // const [answer, setAnswer] = useState({} as IAnswer); + const [done, setDone] = useState(true); + + const read = useCallback( + async ( + body: any, + ): Promise<{ response: Response; data: ResponseType } | undefined> => { + try { + setDone(false); + const response = await fetch(url, { + method: 'POST', + headers: { + [Authorization]: getAuthorization(), + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }); + + const res = response.clone().json(); + + const reader = response?.body?.getReader(); + + while (true) { + const x = await reader?.read(); + if (x) { + const { done, value } = x; + try { + // const val = JSON.parse(value || ''); + const val = value; + // const d = val?.data; + // if (typeof d !== 'boolean') { + // console.info('data:', d); + // setAnswer({ + // ...d, + // conversationId: body?.conversation_id, + // }); + // } + } catch (e) { + console.warn(e); + } + if (done) { + console.info('done'); + break; + } + } + } + console.info('done?'); + setDone(true); + // setAnswer({} as IAnswer); + return { data: await res, response }; + } catch (e) { + setDone(true); + // setAnswer({} as IAnswer); + console.warn(e); + } + }, + [url], + ); + + return { read, done, setDone }; +}; + //#region chat hooks export const useScrollToBottom = (messages?: unknown) => {