pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-21 21:36:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: select the corresponding parsing method according to the file type and after the document is successfully uploaded, use the ChunkMethodModal to select the parsing method. and remove ChunkMethodModal from knowledge-file (#158)

Browse Source 
* feat: select the corresponding parsing method according to the file type

* feat: after the document is successfully uploaded, use the ChunkMethodModal  to select the parsing method.

* feat: add pdf types to ParserListMap

* feat: remove ChunkMethodModal from knowledge-file
This commit is contained in:
balibabu
2024-03-27 17:56:34 +08:00
committed by GitHub
parent bf2e3d7fc1
commit 44541a8c33
14 changed files with 340 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

241
web/src/pages/add-knowledge/components/knowledge-dataset/knowledge-upload-file/index.tsx
View File

@ -1,18 +1,24 @@
import { ReactComponent as SelectFilesEndIcon } from '@/assets/svg/select-files-end.svg';
import { ReactComponent as SelectFilesStartIcon } from '@/assets/svg/select-files-start.svg';
import ChunkMethodModal from '@/components/chunk-method-modal';
import { KnowledgeRouteKey } from '@/constants/knowledge';
import {
useRunDocument,
useSelectDocumentList,
useUploadDocument,
} from '@/hooks/documentHooks';
import {
useDeleteDocumentById,
useFetchKnowledgeDetail,
useGetDocumentDefaultParser,
useKnowledgeBaseId,
} from '@/hooks/knowledgeHook';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import uploadService from '@/services/uploadService';
import { isFileUploadDone } from '@/utils/documentUtils';
useChangeDocumentParser,
useSetSelectedRecord,
} from '@/hooks/logicHooks';
import { useFetchTenantInfo } from '@/hooks/userSettingHook';
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { getExtension, isFileUploadDone } from '@/utils/documentUtils';
import {
ArrowLeftOutlined,
CloudUploadOutlined,
@ -24,27 +30,16 @@ import {
Button,
Card,
Flex,
Popover,
Progress,
Radio,
RadioChangeEvent,
Space,
Upload,
UploadFile,
UploadProps,
} from 'antd';
import classNames from 'classnames';
import {
ReactElement,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react';
import { Link, useDispatch, useNavigate } from 'umi';
import { ReactElement, useCallback, useMemo, useRef, useState } from 'react';
import { Link, useNavigate } from 'umi';
import { useSetDocumentParser } from '@/hooks/documentHooks';
import styles from './index.less';
const { Dragger } = Upload;
@ -57,48 +52,21 @@ const UploaderItem = ({
file,
isUpload,
remove,
handleEdit,
}: {
isUpload: boolean;
originNode: ReactElement;
file: UploadFile;
fileList: object[];
showModal: () => void;
remove: (id: string) => void;
setRecord: (record: IKnowledgeFile) => void;
handleEdit: (id: string) => void;
}) => {
const { parserConfig, defaultParserId } = useGetDocumentDefaultParser();
const { removeDocument } = useDeleteDocumentById();
const [value, setValue] = useState(defaultParserId);
const setDocumentParser = useSetDocumentParser();
const documentId = file?.response?.id;
const parserList = useSelectParserList();
const saveParser = (parserId: string) => {
setDocumentParser(parserId, documentId, parserConfig as any);
};
const onChange = (e: RadioChangeEvent) => {
const val = e.target.value;
setValue(val);
saveParser(val);
};
const content = (
<Radio.Group onChange={onChange} value={value}>
<Space direction="vertical">
{parserList.map(
(
x, // value is lowercase, key is uppercase
) => (
<Radio value={x.value} key={x.value}>
{x.label}
</Radio>
),
)}
</Space>
</Radio.Group>
);
const handleRemove = async () => {
if (file.status === 'error') {
remove(documentId);
@ -110,9 +78,11 @@ const UploaderItem = ({
}
};
useEffect(() => {
setValue(defaultParserId);
}, [defaultParserId]);
const handleEditClick = () => {
if (file.status === 'done') {
handleEdit(documentId);
}
};
return (
<Card className={styles.uploaderItem}>
@ -130,9 +100,7 @@ const UploaderItem = ({
onClick={handleRemove}
/>
) : (
<Popover content={content} placement="bottom">
<EditOutlined />
</Popover>
<EditOutlined onClick={handleEditClick} />
)}
</Flex>
<Flex>
@ -153,10 +121,20 @@ const UploaderItem = ({
const KnowledgeUploadFile = () => {
const knowledgeBaseId = useKnowledgeBaseId();
const [isUpload, setIsUpload] = useState(true);
const dispatch = useDispatch();
const [uploadedFileIds, setUploadedFileIds] = useState<string[]>([]);
const fileListRef = useRef<UploadFile[]>([]);
const navigate = useNavigate();
const { currentRecord, setRecord } = useSetSelectedRecord();
const {
changeParserLoading,
onChangeParserOk,
changeParserVisible,
hideChangeParserModal,
showChangeParserModal,
} = useChangeDocumentParser(currentRecord.id);
const documentList = useSelectDocumentList();
const runDocumentByIds = useRunDocument();
const uploadDocument = useUploadDocument();
const enabled = useMemo(() => {
if (isUpload) {
@ -175,8 +153,7 @@ const KnowledgeUploadFile = () => {
onError,
// onProgress,
}) {
const ret = await uploadService.uploadFile(file, knowledgeBaseId);
const data = ret?.data;
const data = await uploadDocument(file as UploadFile);
if (data?.retcode === 0) {
setUploadedFileIds((pre) => {
return pre.concat(data.data.id);
@ -197,6 +174,17 @@ const KnowledgeUploadFile = () => {
});
}, []);
const handleItemEdit = useCallback(
(id: string) => {
const document = documentList.find((x) => x.id === id);
if (document) {
setRecord(document);
}
showChangeParserModal();
},
[documentList, showChangeParserModal, setRecord],
);
const props: UploadProps = {
name: 'file',
multiple: true,
@ -215,6 +203,9 @@ const KnowledgeUploadFile = () => {
fileList={fileList}
originNode={originNode}
remove={remove}
showModal={showChangeParserModal}
setRecord={setRecord}
handleEdit={handleItemEdit}
></UploaderItem>
);
},
@ -226,10 +217,7 @@ const KnowledgeUploadFile = () => {
const runSelectedDocument = () => {
const ids = fileListRef.current.map((x) => x.response.id);
dispatch({
type: 'kFModel/document_run',
payload: { doc_ids: ids, run: 1 },
});
runDocumentByIds(ids);
};
const handleNextClick = () => {
@ -245,67 +233,78 @@ const KnowledgeUploadFile = () => {
useFetchKnowledgeDetail();
return (
<div className={styles.uploadWrapper}>
<section>
<Space className={styles.backToList}>
<ArrowLeftOutlined />
<Link to={`/knowledge/dataset?id=${knowledgeBaseId}`}>
Back to select files
</Link>
</Space>
<div className={styles.progressWrapper}>
<Flex align="center" justify="center">
<SelectFilesStartIcon></SelectFilesStartIcon>
<Progress
percent={100}
showInfo={false}
className={styles.progress}
strokeColor="
rgba(127, 86, 217, 1)
"
/>
<SelectFilesEndIcon></SelectFilesEndIcon>
</Flex>
<Flex justify="space-around">
<p className={styles.selectFilesText}>
<b>Select files</b>
<>
<div className={styles.uploadWrapper}>
<section>
<Space className={styles.backToList}>
<ArrowLeftOutlined />
<Link to={`/knowledge/dataset?id=${knowledgeBaseId}`}>
Back to select files
</Link>
</Space>
<div className={styles.progressWrapper}>
<Flex align="center" justify="center">
<SelectFilesStartIcon></SelectFilesStartIcon>
<Progress
percent={100}
showInfo={false}
className={styles.progress}
strokeColor="
rgba(127, 86, 217, 1)
"
/>
<SelectFilesEndIcon></SelectFilesEndIcon>
</Flex>
<Flex justify="space-around">
<p className={styles.selectFilesText}>
<b>Select files</b>
</p>
<p className={styles.changeSpecificCategoryText}>
<b>Change specific category</b>
</p>
</Flex>
</div>
</section>
<section className={styles.uploadContent}>
<Dragger
{...props}
className={classNames(styles.uploader, {
[styles.hiddenUploader]: !isUpload,
})}
>
<Button className={styles.uploaderButton}>
<CloudUploadOutlined className={styles.uploaderIcon} />
</Button>
<p className="ant-upload-text">
Click or drag file to this area to upload
</p>
<p className={styles.changeSpecificCategoryText}>
<b>Change specific category</b>
<p className="ant-upload-hint">
Support for a single or bulk upload. Strictly prohibited from
uploading company data or other banned files.
</p>
</Flex>
</div>
</section>
<section className={styles.uploadContent}>
<Dragger
{...props}
className={classNames(styles.uploader, {
[styles.hiddenUploader]: !isUpload,
})}
>
<Button className={styles.uploaderButton}>
<CloudUploadOutlined className={styles.uploaderIcon} />
</Dragger>
</section>
<section className={styles.footer}>
<Button
type="primary"
// className={styles.nextButton}
onClick={handleNextClick}
disabled={!enabled}
>
Next
</Button>
<p className="ant-upload-text">
Click or drag file to this area to upload
</p>
<p className="ant-upload-hint">
Support for a single or bulk upload. Strictly prohibited from
uploading company data or other banned files.
</p>
</Dragger>
</section>
<section className={styles.footer}>
<Button
type="primary"
// className={styles.nextButton}
onClick={handleNextClick}
disabled={!enabled}
>
Next
</Button>
</section>
</div>
</section>
</div>
<ChunkMethodModal
parserId={currentRecord.parser_id}
parserConfig={currentRecord.parser_config}
documentExtension={getExtension(currentRecord.name)}
onOk={onChangeParserOk}
visible={changeParserVisible}
hideModal={hideChangeParserModal}
loading={changeParserLoading}
/>
</>
);
};

278
web/src/pages/add-knowledge/components/knowledge-file/chunk-method-modal.tsx
View File

@ -1,278 +0,0 @@
import MaxTokenNumber from '@/components/max-token-number';
import { IModalManagerChildrenProps } from '@/components/modal-manager';
import { IKnowledgeFileParserConfig } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
import {
MinusCircleOutlined,
PlusOutlined,
QuestionCircleOutlined,
} from '@ant-design/icons';
import {
Button,
Divider,
Form,
InputNumber,
Modal,
Space,
Switch,
Tag,
Tooltip,
} from 'antd';
import omit from 'lodash/omit';
import React, { useEffect, useMemo } from 'react';
import { useFetchParserListOnMount } from './hooks';
import styles from './index.less';
const { CheckableTag } = Tag;
interface IProps extends Omit<IModalManagerChildrenProps, 'showModal'> {
loading: boolean;
onOk: (
parserId: string,
parserConfig: IChangeParserConfigRequestBody,
) => void;
showModal?(): void;
parserId: string;
parserConfig: IKnowledgeFileParserConfig;
documentType: string;
}
const hidePagesChunkMethods = ['qa', 'table', 'picture', 'resume', 'one'];
const ChunkMethodModal: React.FC<IProps> = ({
parserId,
onOk,
hideModal,
visible,
documentType,
parserConfig,
}) => {
const { parserList, handleChange, selectedTag } =
useFetchParserListOnMount(parserId);
const [form] = Form.useForm();
const handleOk = async () => {
const values = await form.validateFields();
const parser_config = {
...values.parser_config,
pages: values.pages?.map((x: any) => [x.from, x.to]) ?? [],
};
onOk(selectedTag, parser_config);
};
const showPages = useMemo(() => {
return (
documentType === 'pdf' &&
hidePagesChunkMethods.every((x) => x !== selectedTag)
);
}, [documentType, selectedTag]);
const showOne = useMemo(() => {
return showPages || selectedTag === 'one';
}, [showPages, selectedTag]);
const afterClose = () => {
form.resetFields();
};
useEffect(() => {
if (visible) {
const pages =
parserConfig.pages?.map((x) => ({ from: x[0], to: x[1] })) ?? [];
form.setFieldsValue({
pages: pages.length > 0 ? pages : [{ from: 1, to: 1024 }],
parser_config: omit(parserConfig, 'pages'),
});
}
}, [form, parserConfig, visible]);
return (
<Modal
title="Chunk Method"
open={visible}
onOk={handleOk}
onCancel={hideModal}
afterClose={afterClose}
>
<Space size={[0, 8]} wrap>
<div className={styles.tags}>
{parserList.map((x) => {
return (
<CheckableTag
key={x.value}
checked={selectedTag === x.value}
onChange={(checked) => {
handleChange(x.value, checked);
}}
>
{x.label}
</CheckableTag>
);
})}
</div>
</Space>
<Divider></Divider>
{
<Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
{showPages && (
<>
<Space>
<p>Page Ranges:</p>
<Tooltip
title={
'page ranges: Define the page ranges that need to be parsed. The pages that not included in these ranges will be ignored.'
}
>
<QuestionCircleOutlined
className={styles.questionIcon}
></QuestionCircleOutlined>
</Tooltip>
</Space>
<Form.List name="pages">
{(fields, { add, remove }) => (
<>
{fields.map(({ key, name, ...restField }) => (
<Space
key={key}
style={{
display: 'flex',
}}
align="baseline"
>
<Form.Item
{...restField}
name={[name, 'from']}
dependencies={name > 0 ? [name - 1, 'to'] : []}
rules={[
{
required: true,
message: 'Missing start page number',
},
({ getFieldValue }) => ({
validator(_, value) {
if (
name === 0 ||
!value ||
getFieldValue(['pages', name - 1, 'to']) <
value
) {
return Promise.resolve();
}
return Promise.reject(
new Error(
'The current value must be greater than the previous to!',
),
);
},
}),
]}
>
<InputNumber
placeholder="from"
min={0}
precision={0}
className={styles.pageInputNumber}
/>
</Form.Item>
<Form.Item
{...restField}
name={[name, 'to']}
dependencies={[name, 'from']}
rules={[
{
required: true,
message: 'Missing end page number(excluded)',
},
({ getFieldValue }) => ({
validator(_, value) {
if (
!value ||
getFieldValue(['pages', name, 'from']) < value
) {
return Promise.resolve();
}
return Promise.reject(
new Error(
'The current value must be greater than to!',
),
);
},
}),
]}
>
<InputNumber
placeholder="to"
min={0}
precision={0}
className={styles.pageInputNumber}
/>
</Form.Item>
{name > 0 && (
<MinusCircleOutlined onClick={() => remove(name)} />
)}
</Space>
))}
<Form.Item>
<Button
type="dashed"
onClick={() => add()}
block
icon={<PlusOutlined />}
>
Add page
</Button>
</Form.Item>
</>
)}
</Form.List>
</>
)}
{showOne && (
<Form.Item
name={['parser_config', 'layout_recognize']}
label="Layout recognize"
initialValue={true}
valuePropName="checked"
tooltip={
'Use visual models for layout analysis to better identify document structure, find where the titles, text blocks, images, and tables are. Without this feature, only the plain text of the PDF can be obtained.'
}
>
<Switch />
</Form.Item>
)}
{showPages && (
<Form.Item
noStyle
dependencies={[['parser_config', 'layout_recognize']]}
>
{({ getFieldValue }) =>
getFieldValue(['parser_config', 'layout_recognize']) && (
<Form.Item
name={['parser_config', 'task_page_size']}
label="Task page size"
tooltip={`If using layout recognize, the PDF file will be split into groups of successive. Layout analysis will be performed parallelly between groups to increase the processing speed.
The 'Task page size' determines the size of groups. The larger the page size is, the lower the chance of splitting continuous text between pages into different chunks.`}
initialValue={12}
rules={[
{
required: true,
message: 'Please input your task page size!',
},
]}
>
<InputNumber min={1} max={128} />
</Form.Item>
)
}
</Form.Item>
)}
{selectedTag === 'naive' && <MaxTokenNumber></MaxTokenNumber>}
</Form>
}
</Modal>
);
};
export default ChunkMethodModal;

23
web/src/pages/add-knowledge/components/knowledge-file/hooks.ts
View File

@ -7,10 +7,7 @@ import {
} from '@/hooks/documentHooks';
import { useGetKnowledgeSearchParams } from '@/hooks/routeHook';
import { useOneNamespaceEffectsLoading } from '@/hooks/storeHooks';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import { useFetchTenantInfo } from '@/hooks/userSettingHook';
import { Pagination } from '@/interfaces/common';
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
@ -243,21 +240,3 @@ export const useChangeDocumentParser = (documentId: string) => {
showChangeParserModal,
};
};
export const useFetchParserListOnMount = (parserId: string) => {
const [selectedTag, setSelectedTag] = useState('');
const parserList = useSelectParserList();
useFetchTenantInfo();
useEffect(() => {
setSelectedTag(parserId);
}, [parserId]);
const handleChange = (tag: string, checked: boolean) => {
const nextSelectedTag = checked ? tag : selectedTag;
setSelectedTag(nextSelectedTag);
};
return { parserList, handleChange, selectedTag };
};

5
web/src/pages/add-knowledge/components/knowledge-file/index.tsx
View File

@ -23,7 +23,6 @@ import {
} from 'antd';
import type { ColumnsType } from 'antd/es/table';
import { useMemo } from 'react';
import ChunkMethodModal from './chunk-method-modal';
import CreateFileModal from './create-file-modal';
import {
useChangeDocumentParser,
@ -39,6 +38,8 @@ import ParsingActionCell from './parsing-action-cell';
import ParsingStatusCell from './parsing-status-cell';
import RenameModal from './rename-modal';
import ChunkMethodModal from '@/components/chunk-method-modal';
import { getExtension } from '@/utils/documentUtils';
import styles from './index.less';
const KnowledgeFile = () => {
@ -227,7 +228,7 @@ const KnowledgeFile = () => {
<ChunkMethodModal
parserId={currentRecord.parser_id}
parserConfig={currentRecord.parser_config}
documentType={currentRecord.type}
documentExtension={getExtension(currentRecord.name)}
onOk={onChangeParserOk}
visible={changeParserVisible}
hideModal={hideChangeParserModal}

13
web/src/pages/add-knowledge/components/knowledge-file/model.ts
View File

@ -209,6 +209,19 @@ const model: DvaModel<KFModelState> = {
console.warn(error);
}
},
*upload_document({ payload = {} }, { call, put }) {
const formData = new FormData();
formData.append('file', payload.file);
formData.append('kb_id', payload.kb_id);
const { data } = yield call(kbService.document_upload, formData);
if (data.retcode === 0) {
yield put({
type: 'getKfList',
payload: { kb_id: payload.kb_id },
});
}
return data;
},
},
};
export default model;

42
web/src/pages/add-knowledge/components/knowledge-file/upload.tsx
View File

@ -1,42 +0,0 @@
import { useKnowledgeBaseId } from '@/hooks/knowledgeHook';
import uploadService from '@/services/uploadService';
import type { UploadProps } from 'antd';
import React from 'react';
import { Link } from 'umi';
interface PropsType {
kb_id: string;
getKfList: () => void;
}
type UploadRequestOption = Parameters<
NonNullable<UploadProps['customRequest']>
>[0];
const FileUpload: React.FC<PropsType> = ({ kb_id, getKfList }) => {
const knowledgeBaseId = useKnowledgeBaseId();
const createRequest: (props: UploadRequestOption) => void = async function ({
file,
onSuccess,
onError,
}) {
const { retcode, data } = await uploadService.uploadFile(file, kb_id);
if (retcode === 0) {
onSuccess && onSuccess(data, file);
} else {
onError && onError(data);
}
getKfList && getKfList();
};
const uploadProps: UploadProps = {
customRequest: createRequest,
showUploadList: false,
};
return (
// <Upload {...uploadProps}>
<Link to={`/knowledge/dataset/upload?id=${knowledgeBaseId}`}>导入文件</Link>
// </Upload>
);
};
export default FileUpload;