diff --git a/api/apps/document_app.py b/api/apps/document_app.py index 21aff7dd6..e96791a2f 100644 --- a/api/apps/document_app.py +++ b/api/apps/document_app.py @@ -71,11 +71,13 @@ def upload(): if not e: raise LookupError("Can't find this knowledgebase!") - err, _ = FileService.upload_document(kb, file_objs, current_user.id) + err, files = FileService.upload_document(kb, file_objs, current_user.id) + files = [f[0] for f in files] # remove the blob + if err: return get_json_result( - data=False, message="\n".join(err), code=settings.RetCode.SERVER_ERROR) - return get_json_result(data=True) + data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR) + return get_json_result(data=files) @manager.route('/web_crawl', methods=['POST']) # noqa: F821 diff --git a/web/src/components/file-upload-modal/index.tsx b/web/src/components/file-upload-modal/index.tsx index 1d827af78..6074fc0a2 100644 --- a/web/src/components/file-upload-modal/index.tsx +++ b/web/src/components/file-upload-modal/index.tsx @@ -2,8 +2,10 @@ import { useTranslate } from '@/hooks/common-hooks'; import { IModalProps } from '@/interfaces/common'; import { InboxOutlined } from '@ant-design/icons'; import { + Checkbox, Flex, Modal, + Progress, Segmented, Tabs, TabsProps, @@ -21,10 +23,12 @@ const FileUpload = ({ directory, fileList, setFileList, + uploadProgress, }: { directory: boolean; fileList: UploadFile[]; setFileList: Dispatch>; + uploadProgress: number; }) => { const { t } = useTranslate('fileManager'); const props: UploadProps = { @@ -35,7 +39,7 @@ const FileUpload = ({ newFileList.splice(index, 1); setFileList(newFileList); }, - beforeUpload: (file) => { + beforeUpload: (file: UploadFile) => { setFileList((pre) => { return [...pre, file]; }); @@ -44,38 +48,59 @@ const FileUpload = ({ }, directory, fileList, + progress: { + strokeWidth: 2, + }, }; return ( - -

- -

-

{t('uploadTitle')}

-

{t('uploadDescription')}

- {false &&

{t('uploadLimit')}

} -
+ <> + + +

+ +

+

{t('uploadTitle')}

+

{t('uploadDescription')}

+ {false &&

{t('uploadLimit')}

} +
+ ); }; +interface IFileUploadModalProps extends IModalProps { + uploadFileList: UploadFile[]; + setUploadFileList: Dispatch>; + uploadProgress: number; + setUploadProgress: Dispatch>; +} + const FileUploadModal = ({ visible, hideModal, loading, onOk: onFileUploadOk, -}: IModalProps) => { + uploadFileList: fileList, + setUploadFileList: setFileList, + uploadProgress, + setUploadProgress, +}: IFileUploadModalProps) => { const { t } = useTranslate('fileManager'); const [value, setValue] = useState('local'); - const [fileList, setFileList] = useState([]); - const [directoryFileList, setDirectoryFileList] = useState([]); + const [parseOnCreation, setParseOnCreation] = useState(false); const clearFileList = () => { setFileList([]); - setDirectoryFileList([]); + setUploadProgress(0); }; const onOk = async () => { - const ret = await onFileUploadOk?.([...fileList, ...directoryFileList]); + if (uploadProgress === 100) { + hideModal?.(); + return; + } + + const ret = await onFileUploadOk?.(parseOnCreation); return ret; }; @@ -92,6 +117,7 @@ const FileUploadModal = ({ directory={false} fileList={fileList} setFileList={setFileList} + uploadProgress={uploadProgress} > ), }, @@ -101,8 +127,9 @@ const FileUploadModal = ({ children: ( ), }, @@ -129,7 +156,15 @@ const FileUploadModal = ({ onChange={setValue} /> {value === 'local' ? ( - + <> + setParseOnCreation(e.target.checked)} + > + {t('parseOnCreation')} + + + ) : ( t('comingSoon', { keyPrefix: 'common' }) )} diff --git a/web/src/hooks/document-hooks.ts b/web/src/hooks/document-hooks.ts index 3a95ea380..b3e1ddfab 100644 --- a/web/src/hooks/document-hooks.ts +++ b/web/src/hooks/document-hooks.ts @@ -248,60 +248,27 @@ export const useUploadNextDocument = () => { } = useMutation({ mutationKey: ['uploadDocument'], mutationFn: async (fileList: UploadFile[]) => { - const partitionedFileList = fileList.reduce( - (acc, cur, index) => { - const partIndex = Math.floor(index / 20); // Uploads 20 documents at a time - if (!acc[partIndex]) { - acc[partIndex] = []; - } - acc[partIndex].push(cur); - return acc; - }, - [], - ); + const formData = new FormData(); + formData.append('kb_id', knowledgeId); + fileList.forEach((file: any) => { + formData.append('file', file); + }); - let allRet = []; - for (const listPart of partitionedFileList) { - const formData = new FormData(); - formData.append('kb_id', knowledgeId); - listPart.forEach((file: any) => { - formData.append('file', file); - }); + try { + const ret = await kbService.document_upload(formData); + const code = get(ret, 'data.code'); - try { - const ret = await kbService.document_upload(formData); - allRet.push(ret); - } catch (error) { - allRet.push({ data: { code: 500 } }); - - const filenames = listPart.map((file: any) => file.name).join(', '); - console.warn(error); - console.warn('Error uploading files:', filenames); + if (code === 0 || code === 500) { + queryClient.invalidateQueries({ queryKey: ['fetchDocumentList'] }); } + return ret?.data; + } catch (error) { + console.warn(error); + return { + code: 500, + message: error + '', + }; } - - const succeed = allRet.every((ret) => get(ret, 'data.code') === 0); - const any500 = allRet.some((ret) => get(ret, 'data.code') === 500); - - if (succeed) { - message.success(i18n.t('message.uploaded')); - } - - if (succeed || any500) { - queryClient.invalidateQueries({ queryKey: ['fetchDocumentList'] }); - } - - const allData = { - code: any500 - ? 500 - : succeed - ? 0 - : allRet.filter((ret) => get(ret, 'data.code') !== 0)[0]?.data - ?.code, - data: succeed, - message: allRet.map((ret) => get(ret, 'data.message')).join('/n'), - }; - return allData; }, }); diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 9a4edbbfe..0884b0723 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -126,9 +126,9 @@ export default { filesSelected: 'Files selected', upload: 'Upload', run: 'Parse', - runningStatus0: 'UNParsed', - runningStatus1: 'Parsing', - runningStatus2: 'CANCEL', + runningStatus0: 'PENDING', + runningStatus1: 'PARSING', + runningStatus2: 'CANCELED', runningStatus3: 'SUCCESS', runningStatus4: 'FAIL', pageRanges: 'Page Ranges', @@ -743,6 +743,7 @@ This auto-tag feature enhances retrieval by adding another layer of domain-speci newFolder: 'New Folder', file: 'File', uploadFile: 'Upload File', + parseOnCreation: 'Parse on creation', directory: 'Directory', uploadTitle: 'Drag and drop your file here to upload', uploadDescription: diff --git a/web/src/locales/es.ts b/web/src/locales/es.ts index 36ba3824f..822fe5c2d 100644 --- a/web/src/locales/es.ts +++ b/web/src/locales/es.ts @@ -480,6 +480,7 @@ export default { newFolder: 'Nueva carpeta', file: 'Archivo', uploadFile: 'Subir archivo', + parseOnCreation: 'Ejecutar en la creación', directory: 'Directorio', uploadTitle: 'Haz clic o arrastra el archivo a esta área para subir', uploadDescription: diff --git a/web/src/locales/id.ts b/web/src/locales/id.ts index 504b6dfc1..179f7ff3f 100644 --- a/web/src/locales/id.ts +++ b/web/src/locales/id.ts @@ -648,6 +648,7 @@ export default { newFolder: 'Folder Baru', file: 'File', uploadFile: 'Unggah File', + parseOnCreation: 'Memparsing saat dibuat', directory: 'Direktori', uploadTitle: 'Klik atau seret file ke area ini untuk mengunggah', uploadDescription: diff --git a/web/src/locales/ja.ts b/web/src/locales/ja.ts index 129560ac1..247f02520 100644 --- a/web/src/locales/ja.ts +++ b/web/src/locales/ja.ts @@ -653,6 +653,7 @@ export default { newFolder: '新しいフォルダ', file: 'ファイル', uploadFile: 'ファイルをアップロード', + parseOnCreation: '作成時に解析', directory: 'ディレクトリ', uploadTitle: 'クリックまたはドラッグしてファイルをアップロード', uploadDescription: diff --git a/web/src/locales/pt-br.ts b/web/src/locales/pt-br.ts index 2bfa167e3..914dd215f 100644 --- a/web/src/locales/pt-br.ts +++ b/web/src/locales/pt-br.ts @@ -639,6 +639,7 @@ export default { newFolder: 'Nova Pasta', file: 'Arquivo', uploadFile: 'Carregar Arquivo', + parseOnCreation: 'Executar na criação', directory: 'Diretório', uploadTitle: 'Clique ou arraste o arquivo para esta área para fazer o upload', diff --git a/web/src/locales/vi.ts b/web/src/locales/vi.ts index aed38cf37..030d9ca23 100644 --- a/web/src/locales/vi.ts +++ b/web/src/locales/vi.ts @@ -707,6 +707,7 @@ export default { newFolder: 'Thư mục mới', file: 'Tệp', uploadFile: 'Tải tệp lên', + parseOnCreation: 'Phân tích khi tạo', directory: 'Thư mục', uploadTitle: 'Nhấp hoặc kéo thả tệp vào khu vực này để tải lên', uploadDescription: diff --git a/web/src/locales/zh-traditional.ts b/web/src/locales/zh-traditional.ts index 427fea23d..42dda617d 100644 --- a/web/src/locales/zh-traditional.ts +++ b/web/src/locales/zh-traditional.ts @@ -708,6 +708,7 @@ export default { pleaseSelect: '請選擇', newFolder: '新建文件夾', uploadFile: '上傳文件', + parseOnCreation: '創建時解析', uploadTitle: '點擊或拖拽文件至此區域即可上傳', uploadDescription: '支持單次或批量上傳。單個檔案大小不超過10MB,最多上傳128份檔案。', diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index a93aec7bc..ea4a6bd1b 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -726,6 +726,7 @@ General:实体和关系提取提示来自 GitHub - microsoft/graphrag:基于 pleaseSelect: '请选择', newFolder: '新建文件夹', uploadFile: '上传文件', + parseOnCreation: '创建时解析', uploadTitle: '点击或拖拽文件至此区域即可上传', uploadDescription: '支持单次或批量上传。 单个文件大小不超过10MB,最多上传128份文件。严禁上传违禁文件。', diff --git a/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts b/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts index c993294aa..c0100b505 100644 --- a/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts +++ b/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts @@ -10,7 +10,6 @@ import { } from '@/hooks/document-hooks'; import { useGetKnowledgeSearchParams } from '@/hooks/route-hook'; import { IChangeParserConfigRequestBody } from '@/interfaces/request/document'; -import { getUnSupportedFilesCount } from '@/utils/document-util'; import { UploadFile } from 'antd'; import { useCallback, useState } from 'react'; import { useNavigate } from 'umi'; @@ -143,29 +142,103 @@ export const useHandleUploadDocument = () => { hideModal: hideDocumentUploadModal, showModal: showDocumentUploadModal, } = useSetModalState(); + const [fileList, setFileList] = useState([]); + const [uploadProgress, setUploadProgress] = useState(0); const { uploadDocument, loading } = useUploadNextDocument(); + const { runDocumentByIds, loading: _ } = useRunNextDocument(); const onDocumentUploadOk = useCallback( - async (fileList: UploadFile[]): Promise => { - if (fileList.length > 0) { - const ret: any = await uploadDocument(fileList); - if (typeof ret?.message !== 'string') { - return; - } - const count = getUnSupportedFilesCount(ret?.message); - /// 500 error code indicates that some file types are not supported - let code = ret?.code; - if ( - ret?.code === 0 || - (ret?.code === 500 && count !== fileList.length) // Some files were not uploaded successfully, but some were uploaded successfully. - ) { - code = 0; - hideDocumentUploadModal(); - } - return code; + async (parseOnCreation: boolean): Promise => { + const processFileGroup = async (filesPart: UploadFile[]) => { + // set status to uploading on files + setFileList( + fileList.map((file) => { + if (!filesPart.includes(file)) { + return file; + } + + let newFile = file; + newFile.status = 'uploading'; + newFile.percent = 1; + return newFile; + }), + ); + + const ret = await uploadDocument(filesPart); + + const files = ret?.data || []; + const succesfulFilenames = files.map((file: any) => file.name); + + // set status to done or error on files (based on response) + setFileList( + fileList.map((file) => { + if (!filesPart.includes(file)) { + return file; + } + + let newFile = file; + newFile.status = succesfulFilenames.includes(file.name) + ? 'done' + : 'error'; + newFile.percent = 100; + newFile.response = ret.message; + return newFile; + }), + ); + + return { + code: ret?.code, + fileIds: files.map((file: any) => file.id), + totalSuccess: succesfulFilenames.length, + }; + }; + + const totalFiles = fileList.length; + + if (totalFiles === 0) { + console.log('No files to upload'); + hideDocumentUploadModal(); + return 0; } + + let totalSuccess = 0; + let codes = []; + let toRunFileIds: any[] = []; + for (let i = 0; i < totalFiles; i += 10) { + setUploadProgress(Math.floor((i / totalFiles) * 100)); + const files = fileList.slice(i, i + 10); + const { + code, + totalSuccess: count, + fileIds, + } = await processFileGroup(files); + codes.push(code); + totalSuccess += count; + toRunFileIds = toRunFileIds.concat(fileIds); + } + + const allSuccess = codes.every((code) => code === 0); + const any500 = codes.some((code) => code === 500); + + let code = 500; + if (allSuccess || (any500 && totalSuccess === totalFiles)) { + code = 0; + hideDocumentUploadModal(); + } + + if (parseOnCreation) { + await runDocumentByIds({ + documentIds: toRunFileIds, + run: 1, + shouldDelete: false, + }); + } + + setUploadProgress(100); + + return code; }, - [uploadDocument, hideDocumentUploadModal], + [uploadDocument, hideDocumentUploadModal, fileList], ); return { @@ -174,6 +247,10 @@ export const useHandleUploadDocument = () => { documentUploadVisible, hideDocumentUploadModal, showDocumentUploadModal, + uploadFileList: fileList, + setUploadFileList: setFileList, + uploadProgress, + setUploadProgress, }; }; diff --git a/web/src/pages/add-knowledge/components/knowledge-file/index.tsx b/web/src/pages/add-knowledge/components/knowledge-file/index.tsx index dd203cc09..1ce7df8cc 100644 --- a/web/src/pages/add-knowledge/components/knowledge-file/index.tsx +++ b/web/src/pages/add-knowledge/components/knowledge-file/index.tsx @@ -69,6 +69,10 @@ const KnowledgeFile = () => { showDocumentUploadModal, onDocumentUploadOk, documentUploadLoading, + uploadFileList, + setUploadFileList, + uploadProgress, + setUploadProgress, } = useHandleUploadDocument(); const { webCrawlUploadVisible, @@ -229,6 +233,10 @@ const KnowledgeFile = () => { hideModal={hideDocumentUploadModal} loading={documentUploadLoading} onOk={onDocumentUploadOk} + uploadFileList={uploadFileList} + setUploadFileList={setUploadFileList} + uploadProgress={uploadProgress} + setUploadProgress={setUploadProgress} >