Implements RAPTOR for better chunking #882 (#883)

### What problem does this PR solve?

Implements RAPTOR for better chunking #882

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
balibabu
2024-05-22 18:04:18 +08:00
committed by GitHub
parent 43412571f7
commit ed184ed87e
7 changed files with 288 additions and 10 deletions

View File

@ -8,6 +8,9 @@ import {
import LayoutRecognize from '@/components/layout-recognize';
import MaxTokenNumber from '@/components/max-token-number';
import ParseConfiguration, {
showRaptorParseConfiguration,
} from '@/components/parse-configuration';
import { useTranslate } from '@/hooks/commonHooks';
import { FormInstance } from 'antd/lib';
import styles from './index.less';
@ -99,15 +102,19 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
{({ getFieldValue }) => {
const parserId = getFieldValue('parser_id');
if (parserId === 'naive') {
return (
<>
<MaxTokenNumber></MaxTokenNumber>
<LayoutRecognize></LayoutRecognize>
</>
);
}
return null;
return (
<>
{parserId === 'naive' && (
<>
<MaxTokenNumber></MaxTokenNumber>
<LayoutRecognize></LayoutRecognize>
</>
)}
{showRaptorParseConfiguration(parserId) && (
<ParseConfiguration></ParseConfiguration>
)}
</>
);
}}
</Form.Item>

View File

@ -62,7 +62,7 @@ export const useFetchKnowledgeConfigurationOnMount = (form: FormInstance) => {
'embd_id',
'parser_id',
'language',
'parser_config.chunk_token_num',
'parser_config',
]),
avatar: fileList,
});