From 44541a8c33f7523040e45162f5701335f09e52d3 Mon Sep 17 00:00:00 2001 From: balibabu Date: Wed, 27 Mar 2024 17:56:34 +0800 Subject: [PATCH] feat: select the corresponding parsing method according to the file type and after the document is successfully uploaded, use the ChunkMethodModal to select the parsing method. and remove ChunkMethodModal from knowledge-file (#158) * feat: select the corresponding parsing method according to the file type * feat: after the document is successfully uploaded, use the ChunkMethodModal to select the parsing method. * feat: add pdf types to ParserListMap * feat: remove ChunkMethodModal from knowledge-file --- .../components/chunk-method-modal/hooks.ts | 82 ++++++ .../components/chunk-method-modal/index.less | 10 + .../chunk-method-modal/index.tsx} | 21 +- web/src/hooks/documentHooks.ts | 45 ++++ web/src/hooks/logicHooks.ts | 47 ++++ .../knowledge-upload-file/index.tsx | 241 +++++++++--------- .../components/knowledge-file/hooks.ts | 23 +- .../components/knowledge-file/index.tsx | 5 +- .../components/knowledge-file/model.ts | 13 + .../components/knowledge-file/upload.tsx | 42 --- web/src/services/kbService.ts | 5 + web/src/services/uploadService.ts | 21 -- web/src/utils/api.ts | 1 + web/src/utils/documentUtils.ts | 3 + 14 files changed, 340 insertions(+), 219 deletions(-) create mode 100644 web/src/components/chunk-method-modal/hooks.ts create mode 100644 web/src/components/chunk-method-modal/index.less rename web/src/{pages/add-knowledge/components/knowledge-file/chunk-method-modal.tsx => components/chunk-method-modal/index.tsx} (96%) create mode 100644 web/src/hooks/logicHooks.ts delete mode 100644 web/src/pages/add-knowledge/components/knowledge-file/upload.tsx delete mode 100644 web/src/services/uploadService.ts diff --git a/web/src/components/chunk-method-modal/hooks.ts b/web/src/components/chunk-method-modal/hooks.ts new file mode 100644 index 000000000..193d3f9c4 --- /dev/null +++ b/web/src/components/chunk-method-modal/hooks.ts @@ -0,0 +1,82 @@ +import { + useFetchTenantInfo, + useSelectParserList, +} from '@/hooks/userSettingHook'; +import { useEffect, useMemo, useState } from 'react'; + +const ParserListMap = new Map([ + [ + ['pdf'], + [ + 'naive', + 'resume', + 'manual', + 'paper', + 'book', + 'laws', + 'presentation', + 'one', + ], + ], + [ + ['doc', 'docx'], + ['naive', 'resume', 'book', 'laws', 'one'], + ], + [ + ['xlsx', 'xls'], + ['naive', 'qa', 'table', 'one'], + ], + [['ppt', 'pptx'], ['presentation']], + [ + ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tif', 'tiff', 'webp', 'svg', 'ico'], + ['picture'], + ], + [['txt'], ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table']], + [['csv'], ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table']], +]); + +const getParserList = ( + values: string[], + parserList: Array<{ + value: string; + label: string; + }>, +) => { + return parserList.filter((x) => values?.some((y) => y === x.value)); +}; + +export const useFetchParserListOnMount = ( + parserId: string, + documentExtension: string, +) => { + const [selectedTag, setSelectedTag] = useState(''); + const parserList = useSelectParserList(); + + const nextParserList = useMemo(() => { + const key = [...ParserListMap.keys()].find((x) => + x.some((y) => y === documentExtension), + ); + if (key) { + const values = ParserListMap.get(key); + return getParserList(values ?? [], parserList); + } + + return getParserList( + ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table'], + parserList, + ); + }, [parserList, documentExtension]); + + useFetchTenantInfo(); + + useEffect(() => { + setSelectedTag(parserId); + }, [parserId]); + + const handleChange = (tag: string, checked: boolean) => { + const nextSelectedTag = checked ? tag : selectedTag; + setSelectedTag(nextSelectedTag); + }; + + return { parserList: nextParserList, handleChange, selectedTag }; +}; diff --git a/web/src/components/chunk-method-modal/index.less b/web/src/components/chunk-method-modal/index.less new file mode 100644 index 000000000..8123b026c --- /dev/null +++ b/web/src/components/chunk-method-modal/index.less @@ -0,0 +1,10 @@ +.pageInputNumber { + width: 220px; +} + +.questionIcon { + margin-inline-start: 4px; + color: rgba(0, 0, 0, 0.45); + cursor: help; + writing-mode: horizontal-tb; +} diff --git a/web/src/pages/add-knowledge/components/knowledge-file/chunk-method-modal.tsx b/web/src/components/chunk-method-modal/index.tsx similarity index 96% rename from web/src/pages/add-knowledge/components/knowledge-file/chunk-method-modal.tsx rename to web/src/components/chunk-method-modal/index.tsx index 0d9a5275b..6a0155ee3 100644 --- a/web/src/pages/add-knowledge/components/knowledge-file/chunk-method-modal.tsx +++ b/web/src/components/chunk-method-modal/index.tsx @@ -35,7 +35,7 @@ interface IProps extends Omit { showModal?(): void; parserId: string; parserConfig: IKnowledgeFileParserConfig; - documentType: string; + documentExtension: string; } const hidePagesChunkMethods = ['qa', 'table', 'picture', 'resume', 'one']; @@ -45,11 +45,13 @@ const ChunkMethodModal: React.FC = ({ onOk, hideModal, visible, - documentType, + documentExtension, parserConfig, }) => { - const { parserList, handleChange, selectedTag } = - useFetchParserListOnMount(parserId); + const { parserList, handleChange, selectedTag } = useFetchParserListOnMount( + parserId, + documentExtension, + ); const [form] = Form.useForm(); const handleOk = async () => { @@ -62,11 +64,8 @@ const ChunkMethodModal: React.FC = ({ }; const showPages = useMemo(() => { - return ( - documentType === 'pdf' && - hidePagesChunkMethods.every((x) => x !== selectedTag) - ); - }, [documentType, selectedTag]); + return hidePagesChunkMethods.every((x) => x !== selectedTag); + }, [selectedTag]); const showOne = useMemo(() => { return showPages || selectedTag === 'one'; @@ -114,7 +113,7 @@ const ChunkMethodModal: React.FC = ({ - { + {documentExtension === 'pdf' && (
{showPages && ( <> @@ -271,7 +270,7 @@ const ChunkMethodModal: React.FC = ({ {selectedTag === 'naive' && } - } + )} ); }; diff --git a/web/src/hooks/documentHooks.ts b/web/src/hooks/documentHooks.ts index 19dae5c72..a0b00852e 100644 --- a/web/src/hooks/documentHooks.ts +++ b/web/src/hooks/documentHooks.ts @@ -2,6 +2,7 @@ import { IChunk, IKnowledgeFile } from '@/interfaces/database/knowledge'; import { IChangeParserConfigRequestBody } from '@/interfaces/request/document'; import { api_host } from '@/utils/api'; import { buildChunkHighlights } from '@/utils/documentUtils'; +import { UploadFile } from 'antd'; import { useCallback, useMemo, useState } from 'react'; import { IHighlight } from 'react-pdf-highlighter'; import { useDispatch, useSelector } from 'umi'; @@ -174,3 +175,47 @@ export const useRemoveDocument = (documentId: string) => { return removeDocument; }; + +export const useUploadDocument = () => { + const dispatch = useDispatch(); + const { knowledgeId } = useGetKnowledgeSearchParams(); + + const uploadDocument = useCallback( + (file: UploadFile) => { + try { + return dispatch({ + type: 'kFModel/upload_document', + payload: { + file, + kb_id: knowledgeId, + }, + }); + } catch (errorInfo) { + console.log('Failed:', errorInfo); + } + }, + [dispatch, knowledgeId], + ); + + return uploadDocument; +}; + +export const useRunDocument = () => { + const dispatch = useDispatch(); + + const runDocumentByIds = useCallback( + (ids: string[]) => { + try { + return dispatch({ + type: 'kFModel/document_run', + payload: { doc_ids: ids, run: 1 }, + }); + } catch (errorInfo) { + console.log('Failed:', errorInfo); + } + }, + [dispatch], + ); + + return runDocumentByIds; +}; diff --git a/web/src/hooks/logicHooks.ts b/web/src/hooks/logicHooks.ts new file mode 100644 index 000000000..42dea0ffd --- /dev/null +++ b/web/src/hooks/logicHooks.ts @@ -0,0 +1,47 @@ +import { IKnowledgeFile } from '@/interfaces/database/knowledge'; +import { IChangeParserConfigRequestBody } from '@/interfaces/request/document'; +import { useCallback, useState } from 'react'; +import { useSetModalState } from './commonHooks'; +import { useSetDocumentParser } from './documentHooks'; +import { useOneNamespaceEffectsLoading } from './storeHooks'; + +export const useChangeDocumentParser = (documentId: string) => { + const setDocumentParser = useSetDocumentParser(); + + const { + visible: changeParserVisible, + hideModal: hideChangeParserModal, + showModal: showChangeParserModal, + } = useSetModalState(); + const loading = useOneNamespaceEffectsLoading('kFModel', [ + 'document_change_parser', + ]); + + const onChangeParserOk = useCallback( + async (parserId: string, parserConfig: IChangeParserConfigRequestBody) => { + const ret = await setDocumentParser(parserId, documentId, parserConfig); + if (ret === 0) { + hideChangeParserModal(); + } + }, + [hideChangeParserModal, setDocumentParser, documentId], + ); + + return { + changeParserLoading: loading, + onChangeParserOk, + changeParserVisible, + hideChangeParserModal, + showChangeParserModal, + }; +}; + +export const useSetSelectedRecord = () => { + const [currentRecord, setCurrentRecord] = useState({} as T); + + const setRecord = (record: T) => { + setCurrentRecord(record); + }; + + return { currentRecord, setRecord }; +}; diff --git a/web/src/pages/add-knowledge/components/knowledge-dataset/knowledge-upload-file/index.tsx b/web/src/pages/add-knowledge/components/knowledge-dataset/knowledge-upload-file/index.tsx index 3e84a8ea4..0a8bb87e0 100644 --- a/web/src/pages/add-knowledge/components/knowledge-dataset/knowledge-upload-file/index.tsx +++ b/web/src/pages/add-knowledge/components/knowledge-dataset/knowledge-upload-file/index.tsx @@ -1,18 +1,24 @@ import { ReactComponent as SelectFilesEndIcon } from '@/assets/svg/select-files-end.svg'; import { ReactComponent as SelectFilesStartIcon } from '@/assets/svg/select-files-start.svg'; +import ChunkMethodModal from '@/components/chunk-method-modal'; import { KnowledgeRouteKey } from '@/constants/knowledge'; +import { + useRunDocument, + useSelectDocumentList, + useUploadDocument, +} from '@/hooks/documentHooks'; import { useDeleteDocumentById, useFetchKnowledgeDetail, - useGetDocumentDefaultParser, useKnowledgeBaseId, } from '@/hooks/knowledgeHook'; import { - useFetchTenantInfo, - useSelectParserList, -} from '@/hooks/userSettingHook'; -import uploadService from '@/services/uploadService'; -import { isFileUploadDone } from '@/utils/documentUtils'; + useChangeDocumentParser, + useSetSelectedRecord, +} from '@/hooks/logicHooks'; +import { useFetchTenantInfo } from '@/hooks/userSettingHook'; +import { IKnowledgeFile } from '@/interfaces/database/knowledge'; +import { getExtension, isFileUploadDone } from '@/utils/documentUtils'; import { ArrowLeftOutlined, CloudUploadOutlined, @@ -24,27 +30,16 @@ import { Button, Card, Flex, - Popover, Progress, - Radio, - RadioChangeEvent, Space, Upload, UploadFile, UploadProps, } from 'antd'; import classNames from 'classnames'; -import { - ReactElement, - useCallback, - useEffect, - useMemo, - useRef, - useState, -} from 'react'; -import { Link, useDispatch, useNavigate } from 'umi'; +import { ReactElement, useCallback, useMemo, useRef, useState } from 'react'; +import { Link, useNavigate } from 'umi'; -import { useSetDocumentParser } from '@/hooks/documentHooks'; import styles from './index.less'; const { Dragger } = Upload; @@ -57,48 +52,21 @@ const UploaderItem = ({ file, isUpload, remove, + handleEdit, }: { isUpload: boolean; originNode: ReactElement; file: UploadFile; fileList: object[]; + showModal: () => void; remove: (id: string) => void; + setRecord: (record: IKnowledgeFile) => void; + handleEdit: (id: string) => void; }) => { - const { parserConfig, defaultParserId } = useGetDocumentDefaultParser(); const { removeDocument } = useDeleteDocumentById(); - const [value, setValue] = useState(defaultParserId); - const setDocumentParser = useSetDocumentParser(); const documentId = file?.response?.id; - const parserList = useSelectParserList(); - - const saveParser = (parserId: string) => { - setDocumentParser(parserId, documentId, parserConfig as any); - }; - - const onChange = (e: RadioChangeEvent) => { - const val = e.target.value; - setValue(val); - saveParser(val); - }; - - const content = ( - - - {parserList.map( - ( - x, // value is lowercase, key is uppercase - ) => ( - - {x.label} - - ), - )} - - - ); - const handleRemove = async () => { if (file.status === 'error') { remove(documentId); @@ -110,9 +78,11 @@ const UploaderItem = ({ } }; - useEffect(() => { - setValue(defaultParserId); - }, [defaultParserId]); + const handleEditClick = () => { + if (file.status === 'done') { + handleEdit(documentId); + } + }; return ( @@ -130,9 +100,7 @@ const UploaderItem = ({ onClick={handleRemove} /> ) : ( - - - + )} @@ -153,10 +121,20 @@ const UploaderItem = ({ const KnowledgeUploadFile = () => { const knowledgeBaseId = useKnowledgeBaseId(); const [isUpload, setIsUpload] = useState(true); - const dispatch = useDispatch(); const [uploadedFileIds, setUploadedFileIds] = useState([]); const fileListRef = useRef([]); const navigate = useNavigate(); + const { currentRecord, setRecord } = useSetSelectedRecord(); + const { + changeParserLoading, + onChangeParserOk, + changeParserVisible, + hideChangeParserModal, + showChangeParserModal, + } = useChangeDocumentParser(currentRecord.id); + const documentList = useSelectDocumentList(); + const runDocumentByIds = useRunDocument(); + const uploadDocument = useUploadDocument(); const enabled = useMemo(() => { if (isUpload) { @@ -175,8 +153,7 @@ const KnowledgeUploadFile = () => { onError, // onProgress, }) { - const ret = await uploadService.uploadFile(file, knowledgeBaseId); - const data = ret?.data; + const data = await uploadDocument(file as UploadFile); if (data?.retcode === 0) { setUploadedFileIds((pre) => { return pre.concat(data.data.id); @@ -197,6 +174,17 @@ const KnowledgeUploadFile = () => { }); }, []); + const handleItemEdit = useCallback( + (id: string) => { + const document = documentList.find((x) => x.id === id); + if (document) { + setRecord(document); + } + showChangeParserModal(); + }, + [documentList, showChangeParserModal, setRecord], + ); + const props: UploadProps = { name: 'file', multiple: true, @@ -215,6 +203,9 @@ const KnowledgeUploadFile = () => { fileList={fileList} originNode={originNode} remove={remove} + showModal={showChangeParserModal} + setRecord={setRecord} + handleEdit={handleItemEdit} > ); }, @@ -226,10 +217,7 @@ const KnowledgeUploadFile = () => { const runSelectedDocument = () => { const ids = fileListRef.current.map((x) => x.response.id); - dispatch({ - type: 'kFModel/document_run', - payload: { doc_ids: ids, run: 1 }, - }); + runDocumentByIds(ids); }; const handleNextClick = () => { @@ -245,67 +233,78 @@ const KnowledgeUploadFile = () => { useFetchKnowledgeDetail(); return ( -
-
- - - - Back to select files - - -
- - - - - - -

- Select files + <> +

+
+ + + + Back to select files + + +
+ + + + + + +

+ Select files +

+

+ Change specific category +

+ +
+
+
+ + +

+ Click or drag file to this area to upload

-

- Change specific category +

+ Support for a single or bulk upload. Strictly prohibited from + uploading company data or other banned files.

- -
-
-
- -
+
+ -

- Click or drag file to this area to upload -

-

- Support for a single or bulk upload. Strictly prohibited from - uploading company data or other banned files. -

- -
-
- -
-
+ + + + ); }; diff --git a/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts b/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts index f560954e9..c52da8d14 100644 --- a/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts +++ b/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts @@ -7,10 +7,7 @@ import { } from '@/hooks/documentHooks'; import { useGetKnowledgeSearchParams } from '@/hooks/routeHook'; import { useOneNamespaceEffectsLoading } from '@/hooks/storeHooks'; -import { - useFetchTenantInfo, - useSelectParserList, -} from '@/hooks/userSettingHook'; +import { useFetchTenantInfo } from '@/hooks/userSettingHook'; import { Pagination } from '@/interfaces/common'; import { IKnowledgeFile } from '@/interfaces/database/knowledge'; import { IChangeParserConfigRequestBody } from '@/interfaces/request/document'; @@ -243,21 +240,3 @@ export const useChangeDocumentParser = (documentId: string) => { showChangeParserModal, }; }; - -export const useFetchParserListOnMount = (parserId: string) => { - const [selectedTag, setSelectedTag] = useState(''); - const parserList = useSelectParserList(); - - useFetchTenantInfo(); - - useEffect(() => { - setSelectedTag(parserId); - }, [parserId]); - - const handleChange = (tag: string, checked: boolean) => { - const nextSelectedTag = checked ? tag : selectedTag; - setSelectedTag(nextSelectedTag); - }; - - return { parserList, handleChange, selectedTag }; -}; diff --git a/web/src/pages/add-knowledge/components/knowledge-file/index.tsx b/web/src/pages/add-knowledge/components/knowledge-file/index.tsx index 18f8475a7..73200c222 100644 --- a/web/src/pages/add-knowledge/components/knowledge-file/index.tsx +++ b/web/src/pages/add-knowledge/components/knowledge-file/index.tsx @@ -23,7 +23,6 @@ import { } from 'antd'; import type { ColumnsType } from 'antd/es/table'; import { useMemo } from 'react'; -import ChunkMethodModal from './chunk-method-modal'; import CreateFileModal from './create-file-modal'; import { useChangeDocumentParser, @@ -39,6 +38,8 @@ import ParsingActionCell from './parsing-action-cell'; import ParsingStatusCell from './parsing-status-cell'; import RenameModal from './rename-modal'; +import ChunkMethodModal from '@/components/chunk-method-modal'; +import { getExtension } from '@/utils/documentUtils'; import styles from './index.less'; const KnowledgeFile = () => { @@ -227,7 +228,7 @@ const KnowledgeFile = () => { = { console.warn(error); } }, + *upload_document({ payload = {} }, { call, put }) { + const formData = new FormData(); + formData.append('file', payload.file); + formData.append('kb_id', payload.kb_id); + const { data } = yield call(kbService.document_upload, formData); + if (data.retcode === 0) { + yield put({ + type: 'getKfList', + payload: { kb_id: payload.kb_id }, + }); + } + return data; + }, }, }; export default model; diff --git a/web/src/pages/add-knowledge/components/knowledge-file/upload.tsx b/web/src/pages/add-knowledge/components/knowledge-file/upload.tsx deleted file mode 100644 index 39a859403..000000000 --- a/web/src/pages/add-knowledge/components/knowledge-file/upload.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import { useKnowledgeBaseId } from '@/hooks/knowledgeHook'; -import uploadService from '@/services/uploadService'; -import type { UploadProps } from 'antd'; -import React from 'react'; -import { Link } from 'umi'; -interface PropsType { - kb_id: string; - getKfList: () => void; -} - -type UploadRequestOption = Parameters< - NonNullable ->[0]; - -const FileUpload: React.FC = ({ kb_id, getKfList }) => { - const knowledgeBaseId = useKnowledgeBaseId(); - - const createRequest: (props: UploadRequestOption) => void = async function ({ - file, - onSuccess, - onError, - }) { - const { retcode, data } = await uploadService.uploadFile(file, kb_id); - if (retcode === 0) { - onSuccess && onSuccess(data, file); - } else { - onError && onError(data); - } - getKfList && getKfList(); - }; - const uploadProps: UploadProps = { - customRequest: createRequest, - showUploadList: false, - }; - return ( - // - 导入文件 - // - ); -}; - -export default FileUpload; diff --git a/web/src/services/kbService.ts b/web/src/services/kbService.ts index 8764c6958..decb8c527 100644 --- a/web/src/services/kbService.ts +++ b/web/src/services/kbService.ts @@ -25,6 +25,7 @@ const { document_rename, document_run, get_document_file, + document_upload, } = api; const methods = { @@ -82,6 +83,10 @@ const methods = { url: document_thumbnails, method: 'get', }, + document_upload: { + url: document_upload, + method: 'post', + }, // chunk管理 chunk_list: { url: chunk_list, diff --git a/web/src/services/uploadService.ts b/web/src/services/uploadService.ts deleted file mode 100644 index 6c4d43a6d..000000000 --- a/web/src/services/uploadService.ts +++ /dev/null @@ -1,21 +0,0 @@ -import api from '@/utils/api'; -import request from '@/utils/request'; - -const { upload } = api; - -const uploadService = { - uploadFile: function (file: any, kb_id: string) { - const formData = new FormData(); - formData.append('file', file); - formData.append('kb_id', kb_id); - - const options = { - method: 'post', - data: formData, - }; - - return request(upload, options); - }, -}; - -export default uploadService; diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts index 3b1ec9be4..e64c1697a 100644 --- a/web/src/utils/api.ts +++ b/web/src/utils/api.ts @@ -45,6 +45,7 @@ export default { document_change_parser: `${api_host}/document/change_parser`, document_thumbnails: `${api_host}/document/thumbnails`, get_document_file: `${api_host}/document/get`, + document_upload: `${api_host}/document/upload`, setDialog: `${api_host}/dialog/set`, getDialog: `${api_host}/dialog/get`, diff --git a/web/src/utils/documentUtils.ts b/web/src/utils/documentUtils.ts index 45e8968ea..27e9964bd 100644 --- a/web/src/utils/documentUtils.ts +++ b/web/src/utils/documentUtils.ts @@ -35,3 +35,6 @@ export const buildChunkHighlights = ( }; export const isFileUploadDone = (file: UploadFile) => file.status === 'done'; + +export const getExtension = (name: string) => + name?.slice(name.lastIndexOf('.') + 1).toLowerCase() ?? '';