feat: Supports chatting with files/images #1880 (#1943)

### What problem does this PR solve?

feat: Supports chatting with files/images #1880

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
balibabu
2024-08-14 17:26:47 +08:00
committed by GitHub
parent 78ed8fe9a5
commit a3a5a9966f
17 changed files with 487 additions and 37 deletions

View File

@ -1,396 +0,0 @@
import { Rect } from '@antv/g';
import {
Badge,
BaseBehavior,
BaseNode,
CommonEvent,
ExtensionCategory,
Graph,
NodeEvent,
Point,
Polyline,
PolylineStyleProps,
register,
subStyleProps,
treeToGraphData,
} from '@antv/g6';
import { TreeData } from '@antv/g6/lib/types';
import isEmpty from 'lodash/isEmpty';
import { useCallback, useEffect, useRef } from 'react';
const rootId = 'root';
const COLORS = [
'#5B8FF9',
'#F6BD16',
'#5AD8A6',
'#945FB9',
'#E86452',
'#6DC8EC',
'#FF99C3',
'#1E9493',
'#FF9845',
'#5D7092',
];
const TreeEvent = {
COLLAPSE_EXPAND: 'collapse-expand',
WHEEL: 'canvas:wheel',
};
class IndentedNode extends BaseNode {
static defaultStyleProps = {
ports: [
{
key: 'in',
placement: 'right-bottom',
},
{
key: 'out',
placement: 'left-bottom',
},
],
} as any;
constructor(options: any) {
Object.assign(options.style, IndentedNode.defaultStyleProps);
super(options);
}
get childrenData() {
return this.attributes.context?.model.getChildrenData(this.id);
}
getKeyStyle(attributes: any) {
const [width, height] = this.getSize(attributes);
const keyStyle = super.getKeyStyle(attributes);
return {
width,
height,
...keyStyle,
fill: 'transparent',
};
}
drawKeyShape(attributes: any, container: any) {
const keyStyle = this.getKeyStyle(attributes);
return this.upsert('key', Rect, keyStyle, container);
}
getLabelStyle(attributes: any) {
if (attributes.label === false || !attributes.labelText) return false;
return subStyleProps(this.getGraphicStyle(attributes), 'label') as any;
}
drawIconArea(attributes: any, container: any) {
const [, h] = this.getSize(attributes);
const iconAreaStyle = {
fill: 'transparent',
height: 30,
width: 12,
x: -6,
y: h,
zIndex: -1,
};
this.upsert('icon-area', Rect, iconAreaStyle, container);
}
forwardEvent(target: any, type: any, listener: any) {
if (target && !Reflect.has(target, '__bind__')) {
Reflect.set(target, '__bind__', true);
target.addEventListener(type, listener);
}
}
getCountStyle(attributes: any) {
const { collapsed, color } = attributes;
if (collapsed) {
const [, height] = this.getSize(attributes);
return {
backgroundFill: color,
cursor: 'pointer',
fill: '#fff',
fontSize: 8,
padding: [0, 10],
text: `${this.childrenData?.length}`,
textAlign: 'center',
y: height + 8,
};
}
return false;
}
drawCountShape(attributes: any, container: any) {
const countStyle = this.getCountStyle(attributes);
const btn = this.upsert('count', Badge, countStyle as any, container);
this.forwardEvent(btn, CommonEvent.CLICK, (event: any) => {
event.stopPropagation();
attributes.context.graph.emit(TreeEvent.COLLAPSE_EXPAND, {
id: this.id,
collapsed: false,
});
});
}
isShowCollapse(attributes: any) {
return (
!attributes.collapsed &&
Array.isArray(this.childrenData) &&
this.childrenData?.length > 0
);
}
getCollapseStyle(attributes: any) {
const { showIcon, color } = attributes;
if (!this.isShowCollapse(attributes)) return false;
const [, height] = this.getSize(attributes);
return {
visibility: showIcon ? 'visible' : 'hidden',
backgroundFill: color,
backgroundHeight: 12,
backgroundWidth: 12,
cursor: 'pointer',
fill: '#fff',
fontFamily: 'iconfont',
fontSize: 8,
text: '\ue6e4',
textAlign: 'center',
x: -1, // half of edge line width
y: height + 8,
};
}
drawCollapseShape(attributes: any, container: any) {
const iconStyle = this.getCollapseStyle(attributes);
const btn = this.upsert(
'collapse-expand',
Badge,
iconStyle as any,
container,
);
this.forwardEvent(btn, CommonEvent.CLICK, (event: any) => {
event.stopPropagation();
attributes.context.graph.emit(TreeEvent.COLLAPSE_EXPAND, {
id: this.id,
collapsed: !attributes.collapsed,
});
});
}
getAddStyle(attributes: any) {
const { collapsed, showIcon } = attributes;
if (collapsed) return false;
const [, height] = this.getSize(attributes);
const color = '#ddd';
const lineWidth = 1;
return {
visibility: showIcon ? 'visible' : 'hidden',
backgroundFill: '#fff',
backgroundHeight: 12,
backgroundLineWidth: lineWidth,
backgroundStroke: color,
backgroundWidth: 12,
cursor: 'pointer',
fill: color,
fontFamily: 'iconfont',
text: '\ue664',
textAlign: 'center',
x: -1,
y: height + (this.isShowCollapse(attributes) ? 22 : 8),
};
}
render(attributes = this.parsedAttributes, container = this) {
super.render(attributes, container);
this.drawCountShape(attributes, container);
this.drawIconArea(attributes, container);
this.drawCollapseShape(attributes, container);
}
}
class IndentedEdge extends Polyline {
getControlPoints(
attributes: Required<PolylineStyleProps>,
sourcePoint: Point,
targetPoint: Point,
) {
const [sx] = sourcePoint;
const [, ty] = targetPoint;
return [[sx, ty]] as any;
}
}
class CollapseExpandTree extends BaseBehavior {
constructor(context: any, options: any) {
super(context, options);
this.bindEvents();
}
update(options: any) {
this.unbindEvents();
super.update(options);
this.bindEvents();
}
bindEvents() {
const { graph } = this.context;
graph.on(NodeEvent.POINTER_ENTER, this.showIcon);
graph.on(NodeEvent.POINTER_LEAVE, this.hideIcon);
graph.on(TreeEvent.COLLAPSE_EXPAND, this.onCollapseExpand);
}
unbindEvents() {
const { graph } = this.context;
graph.off(NodeEvent.POINTER_ENTER, this.showIcon);
graph.off(NodeEvent.POINTER_LEAVE, this.hideIcon);
graph.off(TreeEvent.COLLAPSE_EXPAND, this.onCollapseExpand);
}
status = 'idle';
showIcon = (event: any) => {
this.setIcon(event, true);
};
hideIcon = (event: any) => {
this.setIcon(event, false);
};
setIcon = (event: any, show: boolean) => {
if (this.status !== 'idle') return;
const { target } = event;
const id = target.id;
const { graph, element } = this.context;
graph.updateNodeData([{ id, style: { showIcon: show } }]);
element?.draw({ animation: false, silence: true });
};
onCollapseExpand = async (event: any) => {
this.status = 'busy';
const { id, collapsed } = event;
const { graph } = this.context;
if (collapsed) await graph.collapseElement(id);
else await graph.expandElement(id);
this.status = 'idle';
};
}
register(ExtensionCategory.NODE, 'indented', IndentedNode);
register(ExtensionCategory.EDGE, 'indented', IndentedEdge);
register(
ExtensionCategory.BEHAVIOR,
'collapse-expand-tree',
CollapseExpandTree,
);
interface IProps {
data: TreeData;
show: boolean;
}
const IndentedTree = ({ data, show }: IProps) => {
const containerRef = useRef<HTMLDivElement>(null);
const graphRef = useRef<Graph | null>(null);
const render = useCallback(async (data: TreeData) => {
const graph: Graph = new Graph({
container: containerRef.current!,
x: 60,
node: {
type: 'indented',
style: {
size: (d) => [d.id.length * 6 + 10, 20],
labelBackground: (datum) => datum.id === rootId,
labelBackgroundRadius: 0,
labelBackgroundFill: '#576286',
labelFill: (datum) => (datum.id === rootId ? '#fff' : '#666'),
labelText: (d) => d.style?.labelText || d.id,
labelTextAlign: (datum) => (datum.id === rootId ? 'center' : 'left'),
labelTextBaseline: 'top',
color: (datum: any) => {
const depth = graph.getAncestorsData(datum.id, 'tree').length - 1;
return COLORS[depth % COLORS.length] || '#576286';
},
},
state: {
selected: {
lineWidth: 0,
labelFill: '#40A8FF',
labelBackground: true,
labelFontWeight: 'normal',
labelBackgroundFill: '#e8f7ff',
labelBackgroundRadius: 10,
},
},
},
edge: {
type: 'indented',
style: {
radius: 16,
lineWidth: 2,
sourcePort: 'out',
targetPort: 'in',
stroke: (datum: any) => {
const depth = graph.getAncestorsData(datum.source, 'tree').length;
return COLORS[depth % COLORS.length] || 'black';
},
},
},
layout: {
type: 'indented',
direction: 'LR',
isHorizontal: true,
indent: 40,
getHeight: () => 20,
getVGap: () => 10,
},
behaviors: [
'scroll-canvas',
'collapse-expand-tree',
{
type: 'click-select',
enable: (event: any) =>
event.targetType === 'node' && event.target.id !== rootId,
},
],
});
if (graphRef.current) {
graphRef.current.destroy();
}
graphRef.current = graph;
graph.setData(treeToGraphData(data));
graph.render();
}, []);
useEffect(() => {
if (!isEmpty(data)) {
render(data);
}
}, [render, data]);
return (
<div
id="tree"
ref={containerRef}
style={{
width: '90vw',
height: '80vh',
display: show ? 'block' : 'none',
}}
/>
);
};
export default IndentedTree;

View File

@ -1,9 +1,10 @@
import IndentedTree from '@/components/indented-tree/indented-tree';
import { useFetchKnowledgeGraph } from '@/hooks/chunk-hooks';
import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
import { Flex, Modal, Segmented } from 'antd';
import React, { useEffect, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import ForceGraph from './force-graph';
import IndentedTree from './indented-tree';
import styles from './index.less';
import { isDataExist } from './util';
@ -14,7 +15,8 @@ enum SegmentedValue {
const KnowledgeGraphModal: React.FC = () => {
const [isModalOpen, setIsModalOpen] = useState(false);
const { data } = useFetchKnowledgeGraph();
const { documentId } = useGetKnowledgeSearchParams();
const { data } = useFetchKnowledgeGraph(documentId);
const [value, setValue] = useState<SegmentedValue>(SegmentedValue.Graph);
const { t } = useTranslation();

View File

@ -1,8 +1,7 @@
import MessageItem from '@/components/message-item';
import DocumentPreviewer from '@/components/pdf-previewer';
import { MessageType } from '@/constants/chat';
import { useTranslate } from '@/hooks/common-hooks';
import { Button, Drawer, Flex, Input, Spin } from 'antd';
import { Drawer, Flex, Spin } from 'antd';
import {
useClickDrawer,
useFetchConversationOnMount,
@ -14,6 +13,7 @@ import {
} from '../hooks';
import { buildMessageItemReference } from '../utils';
import MessageInput from '@/components/message-input';
import { useFetchUserInfo } from '@/hooks/user-setting-hooks';
import styles from './index.less';
@ -42,7 +42,6 @@ const ChatContainer = () => {
const sendDisabled = useSendButtonDisabled(value);
useGetFileIcon();
const loading = useSelectConversationLoading();
const { t } = useTranslate('chat');
const { data: userInfo } = useFetchUserInfo();
return (
@ -72,7 +71,16 @@ const ChatContainer = () => {
</div>
<div ref={ref} />
</Flex>
<Input
<MessageInput
disabled={disabled}
sendDisabled={sendDisabled}
sendLoading={sendLoading}
value={value}
onInputChange={handleInputChange}
onPressEnter={handlePressEnter}
conversationId={conversation.id}
></MessageInput>
{/* <Input
size="large"
placeholder={t('sendPlaceholder')}
value={value}
@ -89,7 +97,7 @@ const ChatContainer = () => {
}
onPressEnter={handlePressEnter}
onChange={handleInputChange}
/>
/> */}
</Flex>
<Drawer
title="Document Previewer"

View File

@ -547,7 +547,7 @@ export const useSendMessage = (
const { send, answer, done, setDone } = useSendMessageWithSse();
const sendMessage = useCallback(
async (message: string, id?: string) => {
async (message: string, documentIds: string[], id?: string) => {
const res = await send({
conversation_id: id ?? conversationId,
messages: [
@ -555,6 +555,7 @@ export const useSendMessage = (
{
role: MessageType.User,
content: message,
doc_ids: documentIds,
},
],
});
@ -586,14 +587,14 @@ export const useSendMessage = (
);
const handleSendMessage = useCallback(
async (message: string) => {
async (message: string, documentIds: string[]) => {
if (conversationId !== '') {
sendMessage(message);
return sendMessage(message, documentIds);
} else {
const data = await setConversation(message);
if (data.retcode === 0) {
const id = data.data.id;
sendMessage(message, id);
return sendMessage(message, documentIds, id);
}
}
},
@ -614,15 +615,19 @@ export const useSendMessage = (
}
}, [setDone, conversationId]);
const handlePressEnter = useCallback(() => {
if (trim(value) === '') return;
if (done) {
setValue('');
handleSendMessage(value.trim());
}
addNewestConversation(value);
}, [addNewestConversation, handleSendMessage, done, setValue, value]);
const handlePressEnter = useCallback(
async (documentIds: string[]) => {
if (trim(value) === '') return;
let ret;
if (done) {
setValue('');
ret = await handleSendMessage(value.trim(), documentIds);
}
addNewestConversation(value);
return ret;
},
[addNewestConversation, handleSendMessage, done, setValue, value],
);
return {
handlePressEnter,

View File

@ -2,6 +2,7 @@ import { Graph } from '@antv/g6';
import { useSize } from 'ahooks';
import { useEffect, useRef } from 'react';
import { graphData } from './constant';
import InputWithUpload from './input-upload';
import styles from './index.less';
import { Converter } from './util';
@ -108,4 +109,4 @@ const ForceGraph = () => {
return <div ref={containerRef} className={styles.container} />;
};
export default ForceGraph;
export default InputWithUpload;

View File

@ -0,0 +1,118 @@
import { Authorization } from '@/constants/authorization';
import { getAuthorization } from '@/utils/authorization-util';
import { PlusOutlined } from '@ant-design/icons';
import type { GetProp, UploadFile, UploadProps } from 'antd';
import { Image, Input, Upload } from 'antd';
import { useState } from 'react';
import { useGetChatSearchParams } from '../chat/hooks';
type FileType = Parameters<GetProp<UploadProps, 'beforeUpload'>>[0];
const getBase64 = (file: FileType): Promise<string> =>
new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result as string);
reader.onerror = (error) => reject(error);
});
const InputWithUpload = () => {
const [previewOpen, setPreviewOpen] = useState(false);
const [previewImage, setPreviewImage] = useState('');
const { conversationId } = useGetChatSearchParams();
const [fileList, setFileList] = useState<UploadFile[]>([
{
uid: '-1',
name: 'image.png',
status: 'done',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-2',
name: 'image.png',
status: 'done',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-3',
name: 'image.png',
status: 'done',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-4',
name: 'image.png',
status: 'done',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-xxx',
percent: 50,
name: 'image.png',
status: 'uploading',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-5',
name: 'image.png',
status: 'error',
},
]);
const handlePreview = async (file: UploadFile) => {
if (!file.url && !file.preview) {
file.preview = await getBase64(file.originFileObj as FileType);
}
setPreviewImage(file.url || (file.preview as string));
setPreviewOpen(true);
};
const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) =>
setFileList(newFileList);
const uploadButton = (
<button style={{ border: 0, background: 'none' }} type="button">
<PlusOutlined />
<div style={{ marginTop: 8 }}>Upload</div>
</button>
);
return (
<>
<Input placeholder="Basic usage"></Input>
<Upload
action="/v1/document/upload_and_parse"
listType="picture-card"
fileList={fileList}
onPreview={handlePreview}
onChange={handleChange}
multiple
headers={{ [Authorization]: getAuthorization() }}
data={{ conversation_id: '9e9f7d2453e511efb18efa163e197198' }}
method="post"
>
{fileList.length >= 8 ? null : uploadButton}
</Upload>
{previewImage && (
<Image
wrapperStyle={{ display: 'none' }}
preview={{
visible: previewOpen,
onVisibleChange: (visible) => setPreviewOpen(visible),
afterOpenChange: (visible) => !visible && setPreviewImage(''),
}}
src={previewImage}
/>
)}
</>
);
};
export default () => {
return (
<section style={{ height: 500, width: 400 }}>
<div style={{ height: 200 }}></div>
<InputWithUpload></InputWithUpload>
</section>
);
};