feat: Play audio #2088 (#2200)

### What problem does this PR solve?
feat: Play audio #2088


### Type of change


- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
balibabu
2024-09-03 09:55:19 +08:00
committed by GitHub
parent 97e4eccf03
commit 1a1888ed22
5 changed files with 147 additions and 5 deletions

View File

@ -5,6 +5,7 @@ import {
DeleteOutlined,
DislikeOutlined,
LikeOutlined,
PauseCircleOutlined,
SoundOutlined,
SyncOutlined,
} from '@ant-design/icons';
@ -13,7 +14,7 @@ import { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import SvgIcon from '../svg-icon';
import FeedbackModal from './feedback-modal';
import { useRemoveMessage, useSendFeedback } from './hooks';
import { useRemoveMessage, useSendFeedback, useSpeech } from './hooks';
import PromptModal from './prompt-modal';
interface IProps {
@ -37,6 +38,7 @@ export const AssistantGroupButton = ({
showModal: showPromptModal,
} = useSetModalState();
const { t } = useTranslation();
const { handleRead, ref, isPlaying } = useSpeech(content);
const handleLike = useCallback(() => {
onFeedbackOk({ thumbup: true });
@ -48,10 +50,11 @@ export const AssistantGroupButton = ({
<Radio.Button value="a">
<CopyToClipboard text={content}></CopyToClipboard>
</Radio.Button>
<Radio.Button value="b">
<Radio.Button value="b" onClick={handleRead}>
<Tooltip title={t('chat.read')}>
<SoundOutlined />
{isPlaying ? <PauseCircleOutlined /> : <SoundOutlined />}
</Tooltip>
<audio src="" ref={ref}></audio>
</Radio.Button>
{showLikeButton && (
<>

View File

@ -1,9 +1,10 @@
import { useDeleteMessage, useFeedback } from '@/hooks/chat-hooks';
import { useSetModalState } from '@/hooks/common-hooks';
import { IRemoveMessageById } from '@/hooks/logic-hooks';
import { IRemoveMessageById, useSpeechWithSse } from '@/hooks/logic-hooks';
import { IFeedbackRequestBody } from '@/interfaces/request/chat';
import { getMessagePureId } from '@/utils/chat';
import { useCallback } from 'react';
import { SpeechPlayer } from 'openai-speech-stream-player';
import { useCallback, useEffect, useRef, useState } from 'react';
export const useSendFeedback = (messageId: string) => {
const { visible, hideModal, showModal } = useSetModalState();
@ -50,3 +51,52 @@ export const useRemoveMessage = (
return { onRemoveMessage, loading };
};
export const useSpeech = (content: string) => {
const ref = useRef<HTMLAudioElement>(null);
const { read } = useSpeechWithSse();
const player = useRef<SpeechPlayer>();
const [isPlaying, setIsPlaying] = useState<boolean>(false);
const initialize = useCallback(async () => {
player.current = new SpeechPlayer({
audio: ref.current!,
onPlaying: () => {
setIsPlaying(true);
},
onPause: () => {
setIsPlaying(false);
},
onChunkEnd: () => {},
mimeType: 'audio/mpeg',
});
await player.current.init();
}, []);
const pause = useCallback(() => {
player.current?.pause();
}, []);
const speech = useCallback(async () => {
const response = await read({ text: content });
if (response) {
player?.current?.feedWithResponse(response);
}
}, [read, content]);
const handleRead = useCallback(async () => {
if (isPlaying) {
setIsPlaying(false);
pause();
} else {
setIsPlaying(true);
speech();
}
}, [setIsPlaying, speech, isPlaying, pause]);
useEffect(() => {
initialize();
}, [initialize]);
return { ref, handleRead, isPlaying };
};