feat: save the selected parser to the backend on the upload file page and upload document (#54)

* feat: add pagination to document table

* feat: fetch document list by page

* feat: poll the document list

* feat: upload document

* feat: save the selected parser to the backend on the upload file page

web/src/pages/add-knowledge/components/knowledge-dataset/knowledge-upload-file/index.less

@@ -0,0 +1,65 @@
+.uploadWrapper {
+  display: flex;
+  flex-direction: column;
+  padding: 32px;
+  height: 100%;
+  .backToList {
+    padding-bottom: 60px;
+  }
+  .footer {
+    text-align: right;
+    .nextButton {
+      background-color: @purple;
+    }
+  }
+  .uploadContent {
+    flex: 1;
+    padding-top: 60px;
+  }
+  .uploader {
+    display: block;
+    height: 200px;
+  }
+  .hiddenUploader {
+    :global(.ant-upload-drag) {
+      display: none;
+    }
+  }
+  .fileIcon {
+    font-size: 40px;
+    align-items: end;
+  }
+  .deleteIcon {
+    font-size: 20px;
+  }
+  .uploaderItem {
+    margin-top: 16px;
+    :global(.ant-card-body) {
+      padding: 16px;
+    }
+  }
+  .uploaderItemProgress {
+    padding-left: 50px;
+  }
+  .uploaderItemTextWrapper {
+    flex: 1;
+    text-align: left;
+    padding-left: 10px;
+  }
+}
+
+.progressWrapper {
+  text-align: center;
+}
+
+.progress {
+  width: 50%;
+  margin: 0;
+}
+
+.selectFilesText {
+  transform: translateX(10%);
+}
+.changeSpecificCategoryText {
+  transform: translateX(30%);
+}

web/src/pages/add-knowledge/components/knowledge-dataset/knowledge-upload-file/index.tsx

@@ -1,5 +1,266 @@
+import { ReactComponent as SelectFilesEndIcon } from '@/assets/svg/select-files-end.svg';
+import { ReactComponent as SelectFilesStartIcon } from '@/assets/svg/select-files-start.svg';
+import {
+  useDeleteDocumentById,
+  useGetDocumentDefaultParser,
+  useKnowledgeBaseId,
+} from '@/hooks/knowledgeHook';
+import { ITenantInfo } from '@/interfaces/database/knowledge';
+import uploadService from '@/services/uploadService';
+import {
+  ArrowLeftOutlined,
+  DeleteOutlined,
+  EditOutlined,
+  FileDoneOutlined,
+  InboxOutlined,
+} from '@ant-design/icons';
+import {
+  Button,
+  Card,
+  Flex,
+  Popover,
+  Progress,
+  Radio,
+  RadioChangeEvent,
+  Space,
+  Upload,
+  UploadFile,
+  UploadProps,
+} from 'antd';
+import classNames from 'classnames';
+import { ReactElement, useEffect, useState } from 'react';
+import { Nullable } from 'typings';
+import { Link, useDispatch, useNavigate, useSelector } from 'umi';
+
+import { KnowledgeRouteKey } from '@/constants/knowledge';
+import styles from './index.less';
+
+const { Dragger } = Upload;
+
+type UploadRequestOption = Parameters<
+  NonNullable<UploadProps['customRequest']>
+>[0];
+
+const UploaderItem = ({
+  file,
+  actions,
+  isUpload,
+  parserArray,
+}: {
+  isUpload: boolean;
+  originNode: ReactElement;
+  file: UploadFile;
+  fileList: object[];
+  parserArray: string[];
+  actions: { download: Function; preview: Function; remove: any };
+}) => {
+  const { parserConfig, defaultParserId } = useGetDocumentDefaultParser(
+    file?.response?.kb_id,
+  );
+  const { removeDocument } = useDeleteDocumentById();
+  const [value, setValue] = useState(defaultParserId);
+  const dispatch = useDispatch();
+
+  const documentId = file?.response?.id;
+
+  const onChange = (e: RadioChangeEvent) => {
+    const val = e.target.value;
+    setValue(val);
+    saveParser(val);
+  };
+
+  const content = (
+    <Radio.Group onChange={onChange} value={value}>
+      <Space direction="vertical">
+        {parserArray.map((x) => (
+          <Radio value={x} key={x}>
+            {x}
+          </Radio>
+        ))}
+      </Space>
+    </Radio.Group>
+  );
+
+  const handleRemove = async () => {
+    const ret: any = await removeDocument(documentId);
+    if (ret === 0) {
+      actions?.remove();
+    }
+  };
+
+  const saveParser = (parserId: string) => {
+    dispatch({
+      type: 'kFModel/document_change_parser',
+      payload: {
+        parser_id: parserId,
+        doc_id: documentId,
+        parser_config: parserConfig,
+      },
+    });
+  };
+
+  useEffect(() => {
+    setValue(defaultParserId);
+  }, [defaultParserId]);
+
+  return (
+    <Card className={styles.uploaderItem}>
+      <Flex justify="space-between">
+        <FileDoneOutlined className={styles.fileIcon} />
+        <section className={styles.uploaderItemTextWrapper}>
+          <div>
+            <b>{file.name}</b>
+          </div>
+          <span>{file.size}</span>
+        </section>
+        {isUpload ? (
+          <DeleteOutlined
+            className={styles.deleteIcon}
+            onClick={handleRemove}
+          />
+        ) : (
+          <Popover content={content} placement="bottom">
+            <EditOutlined />
+          </Popover>
+        )}
+      </Flex>
+      <Flex>
+        <Progress
+          showInfo={false}
+          percent={100}
+          className={styles.uploaderItemProgress}
+          strokeColor="
+        rgba(127, 86, 217, 1)
+        "
+        />
+        <span>100%</span>
+      </Flex>
+    </Card>
+  );
+};
+
 const KnowledgeUploadFile = () => {
-  return <div>KnowledgeUploadFile</div>;
+  const knowledgeBaseId = useKnowledgeBaseId();
+  const [isUpload, setIsUpload] = useState(true);
+  const dispatch = useDispatch();
+  const tenantIfo: Nullable<ITenantInfo> = useSelector(
+    (state: any) => state.settingModel.tenantIfo,
+  );
+  const navigate = useNavigate();
+
+  const parserArray = tenantIfo?.parser_ids.split(',') ?? [];
+
+  const createRequest: (props: UploadRequestOption) => void = async function ({
+    file,
+    onSuccess,
+    onError,
+    onProgress,
+  }) {
+    const { data } = await uploadService.uploadFile(file, knowledgeBaseId);
+    if (data.retcode === 0) {
+      onSuccess && onSuccess(data.data);
+    } else {
+      onError && onError(data.data);
+    }
+  };
+
+  const props: UploadProps = {
+    name: 'file',
+    multiple: true,
+    itemRender(originNode, file, fileList, actions) {
+      return (
+        <UploaderItem
+          isUpload={isUpload}
+          file={file}
+          fileList={fileList}
+          originNode={originNode}
+          actions={actions}
+          parserArray={parserArray}
+        ></UploaderItem>
+      );
+    },
+    customRequest: createRequest,
+    onDrop(e) {
+      console.log('Dropped files', e.dataTransfer.files);
+    },
+  };
+
+  const handleNextClick = () => {
+    if (!isUpload) {
+      navigate(`/knowledge/${KnowledgeRouteKey.Dataset}?id=${knowledgeBaseId}`);
+    } else {
+      setIsUpload(false);
+    }
+  };
+
+  useEffect(() => {
+    dispatch({
+      type: 'settingModel/getTenantInfo',
+    });
+  }, []);
+
+  return (
+    <div className={styles.uploadWrapper}>
+      <section>
+        <Space className={styles.backToList}>
+          <ArrowLeftOutlined />
+          <Link to={`/knowledge/dataset?id=${knowledgeBaseId}`}>
+            Back to select files
+          </Link>
+        </Space>
+        <div className={styles.progressWrapper}>
+          <Flex align="center" justify="center">
+            <SelectFilesStartIcon></SelectFilesStartIcon>
+            <Progress
+              percent={100}
+              showInfo={false}
+              className={styles.progress}
+              strokeColor="
+               rgba(127, 86, 217, 1)
+              "
+            />
+            <SelectFilesEndIcon></SelectFilesEndIcon>
+          </Flex>
+          <Flex justify="space-around">
+            <p className={styles.selectFilesText}>
+              <b>Select files</b>
+            </p>
+            <p className={styles.changeSpecificCategoryText}>
+              <b>Change specific category</b>
+            </p>
+          </Flex>
+        </div>
+      </section>
+      <section className={styles.uploadContent}>
+        <Dragger
+          {...props}
+          className={classNames(styles.uploader, {
+            [styles.hiddenUploader]: !isUpload,
+          })}
+        >
+          <p className="ant-upload-drag-icon">
+            <InboxOutlined />
+          </p>
+          <p className="ant-upload-text">
+            Click or drag file to this area to upload
+          </p>
+          <p className="ant-upload-hint">
+            Support for a single or bulk upload. Strictly prohibited from
+            uploading company data or other banned files.
+          </p>
+        </Dragger>
+      </section>
+      <section className={styles.footer}>
+        <Button
+          type="primary"
+          className={styles.nextButton}
+          onClick={handleNextClick}
+        >
+          Next
+        </Button>
+      </section>
+    </div>
+  );
 };
 
 export default KnowledgeUploadFile;

web/src/pages/add-knowledge/components/knowledge-file/index.tsx

@@ -1,5 +1,9 @@
 import { KnowledgeRouteKey } from '@/constants/knowledge';
-import { useKnowledgeBaseId } from '@/hooks/knowledgeHook';
+import {
+  useDeleteDocumentById,
+  useKnowledgeBaseId,
+} from '@/hooks/knowledgeHook';
+import { Pagination } from '@/interfaces/common';
 import { IKnowledgeFile } from '@/interfaces/database/knowledge';
 import { getOneNamespaceEffectsLoading } from '@/utils/stroreUtil';
 import { PlusOutlined, SearchOutlined } from '@ant-design/icons';
@@ -15,8 +19,8 @@ import {
   Tag,
 } from 'antd';
 import type { ColumnsType } from 'antd/es/table';
-import { debounce } from 'lodash';
-import React, { useCallback, useEffect, useMemo, useState } from 'react';
+import { PaginationProps } from 'antd/lib';
+import React, { useEffect, useMemo, useState } from 'react';
 import { useDispatch, useNavigate, useSelector } from 'umi';
 import CreateEPModal from './createEFileModal';
 import styles from './index.less';
@@ -30,50 +34,90 @@ const KnowledgeFile = () => {
   const dispatch = useDispatch();
   const kFModel = useSelector((state: any) => state.kFModel);
   const effects = useSelector((state: any) => state.loading.effects);
-  const { data } = kFModel;
+  const { data, total } = kFModel;
   const knowledgeBaseId = useKnowledgeBaseId();
+  const { removeDocument } = useDeleteDocumentById();
 
   const loading = getOneNamespaceEffectsLoading('kFModel', effects, [
     'getKfList',
     'updateDocumentStatus',
   ]);
-  const [inputValue, setInputValue] = useState('');
   const [doc_id, setDocId] = useState('0');
   const [parser_id, setParserId] = useState('0');
   let navigate = useNavigate();
 
-  const getKfList = (keywords?: string) => {
+  const getKfList = () => {
     const payload = {
       kb_id: knowledgeBaseId,
-      keywords,
     };
-    if (!keywords) {
-      delete payload.keywords;
-    }
+
     dispatch({
       type: 'kFModel/getKfList',
       payload,
     });
   };
 
+  const throttledGetDocumentList = () => {
+    dispatch({
+      type: 'kFModel/throttledGetDocumentList',
+      payload: knowledgeBaseId,
+    });
+  };
+
+  const setPagination = (pageNumber = 1, pageSize?: number) => {
+    const pagination: Pagination = {
+      current: pageNumber,
+    } as Pagination;
+    if (pageSize) {
+      pagination.pageSize = pageSize;
+    }
+    dispatch({
+      type: 'kFModel/setPagination',
+      payload: pagination,
+    });
+  };
+
+  const onPageChange: PaginationProps['onChange'] = (pageNumber, pageSize) => {
+    setPagination(pageNumber, pageSize);
+    getKfList();
+  };
+
+  const pagination: PaginationProps = useMemo(() => {
+    return {
+      showQuickJumper: true,
+      total,
+      showSizeChanger: true,
+      current: kFModel.pagination.currentPage,
+      pageSize: kFModel.pagination.pageSize,
+      pageSizeOptions: [1, 2, 10, 20, 50, 100],
+      onChange: onPageChange,
+    };
+  }, [total, kFModel.pagination]);
+
   useEffect(() => {
     if (knowledgeBaseId) {
       getKfList();
+      dispatch({
+        type: 'kFModel/pollGetDocumentList-start',
+        payload: knowledgeBaseId,
+      });
     }
+    return () => {
+      dispatch({
+        type: 'kFModel/pollGetDocumentList-stop',
+      });
+    };
   }, [knowledgeBaseId]);
 
-  const debounceChange = debounce(getKfList, 300);
-  const debounceCallback = useCallback(
-    (value: string) => debounceChange(value),
-    [],
-  );
   const handleInputChange = (
     e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>,
   ) => {
     const value = e.target.value;
-    setInputValue(value);
-    debounceCallback(e.target.value);
+    dispatch({ type: 'kFModel/setSearchString', payload: value });
+    setPagination();
+    throttledGetDocumentList();
   };
+
   const onChangeStatus = (e: boolean, doc_id: string) => {
     dispatch({
       type: 'kFModel/updateDocumentStatus',
@@ -85,13 +129,7 @@ const KnowledgeFile = () => {
     });
   };
   const onRmDocument = () => {
-    dispatch({
-      type: 'kFModel/document_rm',
-      payload: {
-        doc_id,
-        kb_id: knowledgeBaseId,
-      },
-    });
+    removeDocument(doc_id);
   };
   const showCEFModal = () => {
     dispatch({
@@ -226,7 +264,6 @@ const KnowledgeFile = () => {
       key: 'action',
       render: (_, record) => (
         <ParsingActionCell
-          documentId={doc_id}
           knowledgeBaseId={knowledgeBaseId}
           setDocumentAndParserId={setDocumentAndParserId(record)}
           record={record}
@@ -248,12 +285,12 @@ const KnowledgeFile = () => {
       <div className={styles.filter}>
         <Space>
           <h3>Total</h3>
-          <Tag color="purple">100 files</Tag>
+          <Tag color="purple">{total} files</Tag>
         </Space>
         <Space>
           <Input
             placeholder="Seach your files"
-            value={inputValue}
+            value={kFModel.searchString}
             style={{ width: 220 }}
             allowClear
             onChange={handleInputChange}
@@ -272,7 +309,7 @@ const KnowledgeFile = () => {
         columns={finalColumns}
         dataSource={data}
         loading={loading}
-        pagination={false}
+        pagination={pagination}
         scroll={{ scrollToFirstRowOnChange: true, x: true, y: 'fill' }}
       />
       <CreateEPModal getKfList={getKfList} kb_id={knowledgeBaseId} />

web/src/pages/add-knowledge/components/knowledge-file/model.ts

@@ -1,3 +1,4 @@
+import { BaseState } from '@/interfaces/common';
 import { IKnowledgeFile } from '@/interfaces/database/knowledge';
 import kbService from '@/services/kbService';
 import { message } from 'antd';
@@ -6,14 +7,16 @@ import pick from 'lodash/pick';
 import { Nullable } from 'typings';
 import { DvaModel } from 'umi';
 
-export interface KFModelState {
+export interface KFModelState extends BaseState {
   isShowCEFwModal: boolean;
   isShowTntModal: boolean;
   isShowSegmentSetModal: boolean;
   isShowRenameModal: boolean;
   tenantIfo: any;
   data: IKnowledgeFile[];
+  total: number;
   currentRecord: Nullable<IKnowledgeFile>;
+  searchString: string;
 }
 
 const model: DvaModel<KFModelState> = {
@@ -25,7 +28,13 @@ const model: DvaModel<KFModelState> = {
     isShowRenameModal: false,
     tenantIfo: {},
     data: [],
+    total: 0,
     currentRecord: null,
+    searchString: '',
+    pagination: {
+      current: 1,
+      pageSize: 10,
+    },
   },
   reducers: {
     updateState(state, { payload }) {
@@ -40,6 +49,12 @@ const model: DvaModel<KFModelState> = {
     setCurrentRecord(state, { payload }) {
       return { ...state, currentRecord: payload };
     },
+    setSearchString(state, { payload }) {
+      return { ...state, searchString: payload };
+    },
+    setPagination(state, { payload }) {
+      return { ...state, pagination: { ...state.pagination, ...payload } };
+    },
   },
   subscriptions: {
     setup({ dispatch, history }) {
@@ -69,22 +84,41 @@ const model: DvaModel<KFModelState> = {
         callback && callback(res);
       }
     },
-    *getKfList({ payload = {} }, { call, put }) {
-      const { data, response } = yield call(
-        kbService.get_document_list,
-        payload,
-      );
-      const { retcode, data: res, retmsg } = data;
+    *getKfList({ payload = {} }, { call, put, select }) {
+      const state: KFModelState = yield select((state: any) => state.kFModel);
+      const requestBody = {
+        ...payload,
+        page: state.pagination.current,
+        page_size: state.pagination.pageSize,
+      };
+      if (state.searchString) {
+        requestBody['keywords'] = state.searchString;
+      }
+      const { data } = yield call(kbService.get_document_list, requestBody);
+      const { retcode, data: res } = data;
 
       if (retcode === 0) {
         yield put({
           type: 'updateState',
           payload: {
-            data: res,
+            data: res.docs,
+            total: res.total,
           },
         });
       }
     },
+    throttledGetDocumentList: [
+      function* ({ payload }, { call, put }) {
+        yield put({ type: 'getKfList', payload: { kb_id: payload } });
+      },
+      { type: 'throttle', ms: 1000 }, // TODO: Provide type support for this effect
+    ],
+    pollGetDocumentList: [
+      function* ({ payload }, { call, put }) {
+        yield put({ type: 'getKfList', payload: { kb_id: payload } });
+      },
+      { type: 'poll', delay: 5000 }, // TODO: Provide type support for this effect
+    ],
     *updateDocumentStatus({ payload = {} }, { call, put }) {
       const { data, response } = yield call(
         kbService.document_change_status,
@@ -106,11 +140,12 @@ const model: DvaModel<KFModelState> = {
       const { retcode, data: res, retmsg } = data;
       if (retcode === 0) {
         message.success('删除成功！');
-        put({
+        yield put({
           type: 'getKfList',
           payload: { kb_id: payload.kb_id },
         });
       }
+      return retcode;
     },
     *document_rename({ payload = {} }, { call, put }) {
       const { data } = yield call(

web/src/pages/add-knowledge/components/knowledge-file/parsing-action-cell/index.tsx

@@ -5,19 +5,18 @@ import { Button, Dropdown, MenuProps, Space, Tooltip } from 'antd';
 import { useDispatch } from 'umi';
 
 interface IProps {
-  documentId: string;
   knowledgeBaseId: string;
   record: IKnowledgeFile;
   setDocumentAndParserId: () => void;
 }
 
 const ParsingActionCell = ({
-  documentId,
   knowledgeBaseId,
   record,
   setDocumentAndParserId,
 }: IProps) => {
   const dispatch = useDispatch();
+  const documentId = record.id;
 
   const removeDocument = () => {
     dispatch({

web/src/pages/add-knowledge/components/knowledge-file/segmentSetModal.tsx

@@ -18,8 +18,7 @@ const SegmentSetModal: React.FC<kFProps> = ({
   const kFModel = useSelector((state: any) => state.kFModel);
   const settingModel = useSelector((state: any) => state.settingModel);
   const [selectedTag, setSelectedTag] = useState('');
-  const { tenantIfo = {} } = settingModel;
-  const { parser_ids = '' } = tenantIfo;
+  const parser_ids = settingModel?.tenantIfo?.parser_ids ?? '';
   const { isShowSegmentSetModal } = kFModel;
   const { t } = useTranslation();
 

web/src/pages/add-knowledge/components/knowledge-file/upload.tsx

@@ -1,7 +1,8 @@
+import { useKnowledgeBaseId } from '@/hooks/knowledgeHook';
 import uploadService from '@/services/uploadService';
 import type { UploadProps } from 'antd';
-import { Button, Upload } from 'antd';
 import React from 'react';
+import { Link } from 'umi';
 interface PropsType {
   kb_id: string;
   getKfList: () => void;
@@ -12,6 +13,8 @@ type UploadRequestOption = Parameters<
 >[0];
 
 const FileUpload: React.FC<PropsType> = ({ kb_id, getKfList }) => {
+  const knowledgeBaseId = useKnowledgeBaseId();
+
   const createRequest: (props: UploadRequestOption) => void = async function ({
     file,
     onSuccess,
@@ -30,9 +33,9 @@ const FileUpload: React.FC<PropsType> = ({ kb_id, getKfList }) => {
     showUploadList: false,
   };
   return (
-    <Upload {...uploadProps}>
-      <Button type="link">导入文件</Button>
-    </Upload>
+    // <Upload {...uploadProps}>
+    <Link to={`/knowledge/dataset/upload?id=${knowledgeBaseId}`}>导入文件</Link>
+    // </Upload>
   );
 };
 

web/src/pages/add-knowledge/components/knowledge-sidebar/index.tsx

@@ -75,7 +75,6 @@ const KnowledgeSidebar = () => {
   useEffect(() => {
     const widthSize = () => {
       const width = getWidth();
-      console.log(width);
 
       setWindowWidth(width);
     };
@@ -106,7 +105,7 @@ const KnowledgeSidebar = () => {
             [styles.defaultWidth]: windowWidth.width > 957,
             [styles.minWidth]: windowWidth.width <= 957,
           })}
-          inlineCollapsed={collapsed}
+          // inlineCollapsed={collapsed}
           items={items}
           onSelect={handleSelect}
         />

web/src/pages/add-knowledge/index.less

@@ -13,6 +13,6 @@
   .content {
     background-color: white;
     margin-top: 16px;
-    // flex: 1;
+    flex: 1;
   }
 }

web/src/pages/setting/model.ts

@@ -1,6 +1,8 @@
+import { ITenantInfo } from '@/interfaces/database/knowledge';
 import userService from '@/services/userService';
 import authorizationUtil from '@/utils/authorizationUtil';
 import { message } from 'antd';
+import { Nullable } from 'typings';
 import { DvaModel } from 'umi';
 
 export interface SettingModelState {
@@ -9,7 +11,7 @@ export interface SettingModelState {
   isShowSAKModal: boolean;
   isShowSSModal: boolean;
   llm_factory: string;
-  tenantIfo: any;
+  tenantIfo: Nullable<ITenantInfo>;
   llmInfo: any;
   myLlm: any[];
   factoriesList: any[];
@@ -23,7 +25,7 @@ const model: DvaModel<SettingModelState> = {
     isShowSAKModal: false,
     isShowSSModal: false,
     llm_factory: '',
-    tenantIfo: {},
+    tenantIfo: null,
     llmInfo: {},
     myLlm: [],
     factoriesList: [],
@@ -73,23 +75,11 @@ const model: DvaModel<SettingModelState> = {
       }
     },
     *getTenantInfo({ payload = {} }, { call, put }) {
-      yield put({
-        type: 'updateState',
-        payload: {
-          loading: true,
-        },
-      });
       const { data } = yield call(userService.get_tenant_info, payload);
       const { retcode, data: res } = data;
       // llm_id 对应chat_id
       // asr_id 对应speech2txt
 
-      yield put({
-        type: 'updateState',
-        payload: {
-          loading: false,
-        },
-      });
       if (retcode === 0) {
         res.chat_id = res.llm_id;
         res.speech2text_id = res.asr_id;