feat: Supports chatting with files/images #1880 (#1943)

### What problem does this PR solve?

feat: Supports chatting with files/images #1880

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/components/indented-tree/indented-tree.tsx

@@ -0,0 +1,396 @@
+import { Rect } from '@antv/g';
+import {
+  Badge,
+  BaseBehavior,
+  BaseNode,
+  CommonEvent,
+  ExtensionCategory,
+  Graph,
+  NodeEvent,
+  Point,
+  Polyline,
+  PolylineStyleProps,
+  register,
+  subStyleProps,
+  treeToGraphData,
+} from '@antv/g6';
+import { TreeData } from '@antv/g6/lib/types';
+import isEmpty from 'lodash/isEmpty';
+import { useCallback, useEffect, useRef } from 'react';
+
+const rootId = 'root';
+
+const COLORS = [
+  '#5B8FF9',
+  '#F6BD16',
+  '#5AD8A6',
+  '#945FB9',
+  '#E86452',
+  '#6DC8EC',
+  '#FF99C3',
+  '#1E9493',
+  '#FF9845',
+  '#5D7092',
+];
+
+const TreeEvent = {
+  COLLAPSE_EXPAND: 'collapse-expand',
+  WHEEL: 'canvas:wheel',
+};
+
+class IndentedNode extends BaseNode {
+  static defaultStyleProps = {
+    ports: [
+      {
+        key: 'in',
+        placement: 'right-bottom',
+      },
+      {
+        key: 'out',
+        placement: 'left-bottom',
+      },
+    ],
+  } as any;
+
+  constructor(options: any) {
+    Object.assign(options.style, IndentedNode.defaultStyleProps);
+    super(options);
+  }
+
+  get childrenData() {
+    return this.attributes.context?.model.getChildrenData(this.id);
+  }
+
+  getKeyStyle(attributes: any) {
+    const [width, height] = this.getSize(attributes);
+    const keyStyle = super.getKeyStyle(attributes);
+    return {
+      width,
+      height,
+      ...keyStyle,
+      fill: 'transparent',
+    };
+  }
+
+  drawKeyShape(attributes: any, container: any) {
+    const keyStyle = this.getKeyStyle(attributes);
+    return this.upsert('key', Rect, keyStyle, container);
+  }
+
+  getLabelStyle(attributes: any) {
+    if (attributes.label === false || !attributes.labelText) return false;
+    return subStyleProps(this.getGraphicStyle(attributes), 'label') as any;
+  }
+
+  drawIconArea(attributes: any, container: any) {
+    const [, h] = this.getSize(attributes);
+    const iconAreaStyle = {
+      fill: 'transparent',
+      height: 30,
+      width: 12,
+      x: -6,
+      y: h,
+      zIndex: -1,
+    };
+    this.upsert('icon-area', Rect, iconAreaStyle, container);
+  }
+
+  forwardEvent(target: any, type: any, listener: any) {
+    if (target && !Reflect.has(target, '__bind__')) {
+      Reflect.set(target, '__bind__', true);
+      target.addEventListener(type, listener);
+    }
+  }
+
+  getCountStyle(attributes: any) {
+    const { collapsed, color } = attributes;
+    if (collapsed) {
+      const [, height] = this.getSize(attributes);
+      return {
+        backgroundFill: color,
+        cursor: 'pointer',
+        fill: '#fff',
+        fontSize: 8,
+        padding: [0, 10],
+        text: `${this.childrenData?.length}`,
+        textAlign: 'center',
+        y: height + 8,
+      };
+    }
+
+    return false;
+  }
+
+  drawCountShape(attributes: any, container: any) {
+    const countStyle = this.getCountStyle(attributes);
+    const btn = this.upsert('count', Badge, countStyle as any, container);
+
+    this.forwardEvent(btn, CommonEvent.CLICK, (event: any) => {
+      event.stopPropagation();
+      attributes.context.graph.emit(TreeEvent.COLLAPSE_EXPAND, {
+        id: this.id,
+        collapsed: false,
+      });
+    });
+  }
+
+  isShowCollapse(attributes: any) {
+    return (
+      !attributes.collapsed &&
+      Array.isArray(this.childrenData) &&
+      this.childrenData?.length > 0
+    );
+  }
+
+  getCollapseStyle(attributes: any) {
+    const { showIcon, color } = attributes;
+    if (!this.isShowCollapse(attributes)) return false;
+    const [, height] = this.getSize(attributes);
+    return {
+      visibility: showIcon ? 'visible' : 'hidden',
+      backgroundFill: color,
+      backgroundHeight: 12,
+      backgroundWidth: 12,
+      cursor: 'pointer',
+      fill: '#fff',
+      fontFamily: 'iconfont',
+      fontSize: 8,
+      text: '\ue6e4',
+      textAlign: 'center',
+      x: -1, // half of edge line width
+      y: height + 8,
+    };
+  }
+
+  drawCollapseShape(attributes: any, container: any) {
+    const iconStyle = this.getCollapseStyle(attributes);
+    const btn = this.upsert(
+      'collapse-expand',
+      Badge,
+      iconStyle as any,
+      container,
+    );
+
+    this.forwardEvent(btn, CommonEvent.CLICK, (event: any) => {
+      event.stopPropagation();
+      attributes.context.graph.emit(TreeEvent.COLLAPSE_EXPAND, {
+        id: this.id,
+        collapsed: !attributes.collapsed,
+      });
+    });
+  }
+
+  getAddStyle(attributes: any) {
+    const { collapsed, showIcon } = attributes;
+    if (collapsed) return false;
+    const [, height] = this.getSize(attributes);
+    const color = '#ddd';
+    const lineWidth = 1;
+
+    return {
+      visibility: showIcon ? 'visible' : 'hidden',
+      backgroundFill: '#fff',
+      backgroundHeight: 12,
+      backgroundLineWidth: lineWidth,
+      backgroundStroke: color,
+      backgroundWidth: 12,
+      cursor: 'pointer',
+      fill: color,
+      fontFamily: 'iconfont',
+      text: '\ue664',
+      textAlign: 'center',
+      x: -1,
+      y: height + (this.isShowCollapse(attributes) ? 22 : 8),
+    };
+  }
+
+  render(attributes = this.parsedAttributes, container = this) {
+    super.render(attributes, container);
+
+    this.drawCountShape(attributes, container);
+
+    this.drawIconArea(attributes, container);
+    this.drawCollapseShape(attributes, container);
+  }
+}
+
+class IndentedEdge extends Polyline {
+  getControlPoints(
+    attributes: Required<PolylineStyleProps>,
+    sourcePoint: Point,
+    targetPoint: Point,
+  ) {
+    const [sx] = sourcePoint;
+    const [, ty] = targetPoint;
+    return [[sx, ty]] as any;
+  }
+}
+
+class CollapseExpandTree extends BaseBehavior {
+  constructor(context: any, options: any) {
+    super(context, options);
+    this.bindEvents();
+  }
+
+  update(options: any) {
+    this.unbindEvents();
+    super.update(options);
+    this.bindEvents();
+  }
+
+  bindEvents() {
+    const { graph } = this.context;
+
+    graph.on(NodeEvent.POINTER_ENTER, this.showIcon);
+    graph.on(NodeEvent.POINTER_LEAVE, this.hideIcon);
+    graph.on(TreeEvent.COLLAPSE_EXPAND, this.onCollapseExpand);
+  }
+
+  unbindEvents() {
+    const { graph } = this.context;
+
+    graph.off(NodeEvent.POINTER_ENTER, this.showIcon);
+    graph.off(NodeEvent.POINTER_LEAVE, this.hideIcon);
+    graph.off(TreeEvent.COLLAPSE_EXPAND, this.onCollapseExpand);
+  }
+
+  status = 'idle';
+
+  showIcon = (event: any) => {
+    this.setIcon(event, true);
+  };
+
+  hideIcon = (event: any) => {
+    this.setIcon(event, false);
+  };
+
+  setIcon = (event: any, show: boolean) => {
+    if (this.status !== 'idle') return;
+    const { target } = event;
+    const id = target.id;
+    const { graph, element } = this.context;
+    graph.updateNodeData([{ id, style: { showIcon: show } }]);
+    element?.draw({ animation: false, silence: true });
+  };
+
+  onCollapseExpand = async (event: any) => {
+    this.status = 'busy';
+    const { id, collapsed } = event;
+    const { graph } = this.context;
+    if (collapsed) await graph.collapseElement(id);
+    else await graph.expandElement(id);
+    this.status = 'idle';
+  };
+}
+
+register(ExtensionCategory.NODE, 'indented', IndentedNode);
+register(ExtensionCategory.EDGE, 'indented', IndentedEdge);
+register(
+  ExtensionCategory.BEHAVIOR,
+  'collapse-expand-tree',
+  CollapseExpandTree,
+);
+
+interface IProps {
+  data: TreeData;
+  show: boolean;
+}
+
+const IndentedTree = ({ data, show }: IProps) => {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const graphRef = useRef<Graph | null>(null);
+
+  const render = useCallback(async (data: TreeData) => {
+    const graph: Graph = new Graph({
+      container: containerRef.current!,
+      x: 60,
+      node: {
+        type: 'indented',
+        style: {
+          size: (d) => [d.id.length * 6 + 10, 20],
+          labelBackground: (datum) => datum.id === rootId,
+          labelBackgroundRadius: 0,
+          labelBackgroundFill: '#576286',
+          labelFill: (datum) => (datum.id === rootId ? '#fff' : '#666'),
+          labelText: (d) => d.style?.labelText || d.id,
+          labelTextAlign: (datum) => (datum.id === rootId ? 'center' : 'left'),
+          labelTextBaseline: 'top',
+          color: (datum: any) => {
+            const depth = graph.getAncestorsData(datum.id, 'tree').length - 1;
+            return COLORS[depth % COLORS.length] || '#576286';
+          },
+        },
+        state: {
+          selected: {
+            lineWidth: 0,
+            labelFill: '#40A8FF',
+            labelBackground: true,
+            labelFontWeight: 'normal',
+            labelBackgroundFill: '#e8f7ff',
+            labelBackgroundRadius: 10,
+          },
+        },
+      },
+      edge: {
+        type: 'indented',
+        style: {
+          radius: 16,
+          lineWidth: 2,
+          sourcePort: 'out',
+          targetPort: 'in',
+          stroke: (datum: any) => {
+            const depth = graph.getAncestorsData(datum.source, 'tree').length;
+            return COLORS[depth % COLORS.length] || 'black';
+          },
+        },
+      },
+      layout: {
+        type: 'indented',
+        direction: 'LR',
+        isHorizontal: true,
+        indent: 40,
+        getHeight: () => 20,
+        getVGap: () => 10,
+      },
+      behaviors: [
+        'scroll-canvas',
+        'collapse-expand-tree',
+        {
+          type: 'click-select',
+          enable: (event: any) =>
+            event.targetType === 'node' && event.target.id !== rootId,
+        },
+      ],
+    });
+
+    if (graphRef.current) {
+      graphRef.current.destroy();
+    }
+
+    graphRef.current = graph;
+
+    graph.setData(treeToGraphData(data));
+
+    graph.render();
+  }, []);
+
+  useEffect(() => {
+    if (!isEmpty(data)) {
+      render(data);
+    }
+  }, [render, data]);
+
+  return (
+    <div
+      id="tree"
+      ref={containerRef}
+      style={{
+        width: '90vw',
+        height: '80vh',
+        display: show ? 'block' : 'none',
+      }}
+    />
+  );
+};
+
+export default IndentedTree;

web/src/components/indented-tree/modal.tsx

@@ -0,0 +1,31 @@
+import { useFetchKnowledgeGraph } from '@/hooks/chunk-hooks';
+import { Modal } from 'antd';
+import { useTranslation } from 'react-i18next';
+import IndentedTree from './indented-tree';
+
+import { IModalProps } from '@/interfaces/common';
+
+const IndentedTreeModal = ({
+  documentId,
+  visible,
+  hideModal,
+}: IModalProps<any> & { documentId: string }) => {
+  const { data } = useFetchKnowledgeGraph(documentId);
+  const { t } = useTranslation();
+
+  return (
+    <Modal
+      title={t('chunk.graph')}
+      open={visible}
+      onCancel={hideModal}
+      width={'90vw'}
+      footer={null}
+    >
+      <section>
+        <IndentedTree data={data?.data?.mind_map} show></IndentedTree>
+      </section>
+    </Modal>
+  );
+};
+
+export default IndentedTreeModal;

web/src/components/message-input/index.tsx

@@ -0,0 +1,129 @@
+import { Authorization } from '@/constants/authorization';
+import { useTranslate } from '@/hooks/common-hooks';
+import { getAuthorization } from '@/utils/authorization-util';
+import { PlusOutlined } from '@ant-design/icons';
+import type { GetProp, UploadFile } from 'antd';
+import { Button, Flex, Input, Upload, UploadProps } from 'antd';
+import get from 'lodash/get';
+import { ChangeEventHandler, useCallback, useState } from 'react';
+
+type FileType = Parameters<GetProp<UploadProps, 'beforeUpload'>>[0];
+
+interface IProps {
+  disabled: boolean;
+  value: string;
+  sendDisabled: boolean;
+  sendLoading: boolean;
+  onPressEnter(documentIds: string[]): Promise<any>;
+  onInputChange: ChangeEventHandler<HTMLInputElement>;
+  conversationId: string;
+}
+
+const getBase64 = (file: FileType): Promise<string> =>
+  new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(file as any);
+    reader.onload = () => resolve(reader.result as string);
+    reader.onerror = (error) => reject(error);
+  });
+
+const MessageInput = ({
+  disabled,
+  value,
+  onPressEnter,
+  sendDisabled,
+  sendLoading,
+  onInputChange,
+  conversationId,
+}: IProps) => {
+  const { t } = useTranslate('chat');
+
+  const [fileList, setFileList] = useState<UploadFile[]>([
+    // {
+    //   uid: '-1',
+    //   name: 'image.png',
+    //   status: 'done',
+    //   url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
+    // },
+    // {
+    //   uid: '-xxx',
+    //   percent: 50,
+    //   name: 'image.png',
+    //   status: 'uploading',
+    //   url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
+    // },
+    // {
+    //   uid: '-5',
+    //   name: 'image.png',
+    //   status: 'error',
+    // },
+  ]);
+
+  const handlePreview = async (file: UploadFile) => {
+    if (!file.url && !file.preview) {
+      file.preview = await getBase64(file.originFileObj as FileType);
+    }
+
+    // setPreviewImage(file.url || (file.preview as string));
+    // setPreviewOpen(true);
+  };
+
+  const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) => {
+    console.log('🚀 ~ newFileList:', newFileList);
+    setFileList(newFileList);
+  };
+
+  const handlePressEnter = useCallback(async () => {
+    const ids = fileList.reduce((pre, cur) => {
+      return pre.concat(get(cur, 'response.data', []));
+    }, []);
+
+    await onPressEnter(ids);
+    setFileList([]);
+  }, [fileList, onPressEnter]);
+
+  const uploadButton = (
+    <button style={{ border: 0, background: 'none' }} type="button">
+      <PlusOutlined />
+      <div style={{ marginTop: 8 }}>Upload</div>
+    </button>
+  );
+
+  return (
+    <Flex gap={10} vertical>
+      <Input
+        size="large"
+        placeholder={t('sendPlaceholder')}
+        value={value}
+        disabled={disabled}
+        suffix={
+          <Button
+            type="primary"
+            onClick={handlePressEnter}
+            loading={sendLoading}
+            disabled={sendDisabled}
+          >
+            {t('send')}
+          </Button>
+        }
+        onPressEnter={handlePressEnter}
+        onChange={onInputChange}
+      />
+      <Upload
+        action="/v1/document/upload_and_parse"
+        listType="picture-card"
+        fileList={fileList}
+        onPreview={handlePreview}
+        onChange={handleChange}
+        multiple
+        headers={{ [Authorization]: getAuthorization() }}
+        data={{ conversation_id: conversationId }}
+        method="post"
+      >
+        {fileList.length >= 8 ? null : uploadButton}
+      </Upload>
+    </Flex>
+  );
+};
+
+export default MessageInput;

web/src/components/message-item/index.tsx

@@ -1,15 +1,17 @@
 import { ReactComponent as AssistantIcon } from '@/assets/svg/assistant.svg';
 import { MessageType } from '@/constants/chat';
-import { useTranslate } from '@/hooks/common-hooks';
+import { useSetModalState, useTranslate } from '@/hooks/common-hooks';
 import { useSelectFileThumbnails } from '@/hooks/knowledge-hooks';
 import { IReference, Message } from '@/interfaces/database/chat';
 import { IChunk } from '@/interfaces/database/knowledge';
 import classNames from 'classnames';
-import { useMemo } from 'react';
+import { memo, useCallback, useEffect, useMemo, useState } from 'react';
 
+import { useFetchDocumentInfosByIds } from '@/hooks/document-hooks';
 import MarkdownContent from '@/pages/chat/markdown-content';
-import { getExtension } from '@/utils/document-util';
-import { Avatar, Flex, List } from 'antd';
+import { getExtension, isImage } from '@/utils/document-util';
+import { Avatar, Button, Flex, List } from 'antd';
+import IndentedTreeModal from '../indented-tree/modal';
 import NewDocumentLink from '../new-document-link';
 import SvgIcon from '../svg-icon';
 import styles from './index.less';
@@ -32,8 +34,13 @@ const MessageItem = ({
   clickDocumentButton,
 }: IProps) => {
   const isAssistant = item.role === MessageType.Assistant;
+  const isUser = item.role === MessageType.User;
   const { t } = useTranslate('chat');
   const fileThumbnails = useSelectFileThumbnails();
+  const { data: documentList, setDocumentIds } = useFetchDocumentInfosByIds();
+  console.log('🚀 ~ documentList:', documentList);
+  const { visible, hideModal, showModal } = useSetModalState();
+  const [clickedDocumentId, setClickedDocumentId] = useState('');
 
   const referenceDocumentList = useMemo(() => {
     return reference?.doc_aggs ?? [];
@@ -47,6 +54,21 @@ const MessageItem = ({
     return loading ? text?.concat('~~2$$') : text;
   }, [item.content, loading, t]);
 
+  const handleUserDocumentClick = useCallback(
+    (id: string) => () => {
+      setClickedDocumentId(id);
+      showModal();
+    },
+    [showModal],
+  );
+
+  useEffect(() => {
+    const ids = item?.doc_ids ?? [];
+    if (ids.length) {
+      setDocumentIds(ids);
+    }
+  }, [item.doc_ids, setDocumentIds]);
+
   return (
     <div
       className={classNames(styles.messageItem, {
@@ -124,11 +146,62 @@ const MessageItem = ({
                 }}
               />
             )}
+            {isUser && documentList.length > 0 && (
+              <List
+                bordered
+                dataSource={documentList}
+                renderItem={(item) => {
+                  const fileThumbnail = fileThumbnails[item.id];
+                  const fileExtension = getExtension(item.name);
+                  return (
+                    <List.Item>
+                      <Flex gap={'small'} align="center">
+                        {fileThumbnail ? (
+                          <img
+                            src={fileThumbnail}
+                            className={styles.thumbnailImg}
+                          ></img>
+                        ) : (
+                          <SvgIcon
+                            name={`file-icon/${fileExtension}`}
+                            width={24}
+                          ></SvgIcon>
+                        )}
+
+                        {isImage(fileExtension) ? (
+                          <NewDocumentLink
+                            documentId={item.id}
+                            documentName={item.name}
+                            prefix="document"
+                          >
+                            {item.name}
+                          </NewDocumentLink>
+                        ) : (
+                          <Button
+                            type={'text'}
+                            onClick={handleUserDocumentClick(item.id)}
+                          >
+                            {item.name}
+                          </Button>
+                        )}
+                      </Flex>
+                    </List.Item>
+                  );
+                }}
+              />
+            )}
           </Flex>
         </div>
       </section>
+      {visible && (
+        <IndentedTreeModal
+          visible={visible}
+          hideModal={hideModal}
+          documentId={clickedDocumentId}
+        ></IndentedTreeModal>
+      )}
     </div>
   );
 };
 
-export default MessageItem;
+export default memo(MessageItem);