mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-12-20 12:56:55 +08:00
我已在下面的评论中用中文重复说明。 ### What problem does this PR solve? ## Summary This PR enhances the MinerU document parser with additional configuration options, giving users more control over PDF parsing behavior and improving support for multilingual documents. ## Changes ### Backend (`deepdoc/parser/mineru_parser.py`) - Added configurable parsing options: - **Parse Method**: `auto`, `txt`, or `ocr` — allows users to choose the extraction strategy - **Formula Recognition**: Toggle for enabling/disabling formula extraction (useful to disable for Cyrillic documents where it may cause issues) - **Table Recognition**: Toggle for enabling/disabling table extraction - Added language code mapping (`LANGUAGE_TO_MINERU_MAP`) to translate RAGFlow language settings to MinerU-compatible language codes for better OCR accuracy - Improved parser configuration handling to pass these options through the processing pipeline ### Frontend (`web/`) - Created new `MinerUOptionsFormField` component that conditionally renders when MinerU is selected as the layout recognition engine - Added UI controls for: - Parse method selection (dropdown) - Formula recognition toggle (switch) - Table recognition toggle (switch) - Added i18n translations for English and Chinese - Integrated the options into both the dataset creation dialog and dataset settings page ### Integration - Updated `rag/app/naive.py` to forward MinerU options to the parser - Updated task service to handle the new configuration parameters ## Why MinerU is a powerful document parser, but the default settings don't work well for all document types. This PR allows users to: 1. Choose the best parsing method for their documents 2. Disable formula recognition for Cyrillic/non-Latin scripts where it causes issues 3. Control table extraction based on document needs 4. Benefit from automatic language detection for better OCR results ## Testing - [x] Tested MinerU parsing with different parse methods - [x] Verified UI renders correctly when MinerU is selected/deselected - [x] Confirmed settings persist correctly in dataset configuration ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [x] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): --------- Co-authored-by: user210 <user210@rt> Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>
216 lines
5.7 KiB
TypeScript
216 lines
5.7 KiB
TypeScript
import { DataFlowSelect } from '@/components/data-pipeline-select';
|
|
import { ButtonLoading } from '@/components/ui/button';
|
|
import {
|
|
Dialog,
|
|
DialogContent,
|
|
DialogFooter,
|
|
DialogHeader,
|
|
DialogTitle,
|
|
} from '@/components/ui/dialog';
|
|
import {
|
|
Form,
|
|
FormControl,
|
|
FormField,
|
|
FormItem,
|
|
FormLabel,
|
|
FormMessage,
|
|
} from '@/components/ui/form';
|
|
import { Input } from '@/components/ui/input';
|
|
import {
|
|
Select,
|
|
SelectContent,
|
|
SelectItem,
|
|
SelectTrigger,
|
|
SelectValue,
|
|
} from '@/components/ui/select';
|
|
import { LanguageTranslationMap } from '@/constants/common';
|
|
import { FormLayout } from '@/constants/form';
|
|
import { IModalProps } from '@/interfaces/common';
|
|
import { zodResolver } from '@hookform/resolvers/zod';
|
|
import { useEffect, useMemo } from 'react';
|
|
import { useForm, useWatch } from 'react-hook-form';
|
|
import { useTranslation } from 'react-i18next';
|
|
import { z } from 'zod';
|
|
import {
|
|
ChunkMethodItem,
|
|
EmbeddingModelItem,
|
|
ParseTypeItem,
|
|
} from '../dataset/dataset-setting/configuration/common-item';
|
|
|
|
const FormId = 'dataset-creating-form';
|
|
|
|
export function InputForm({ onOk }: IModalProps<any>) {
|
|
const { t } = useTranslation();
|
|
|
|
const languageOptions = useMemo(() => {
|
|
return Object.keys(LanguageTranslationMap).map((x) => ({
|
|
label: x,
|
|
value: x,
|
|
}));
|
|
}, []);
|
|
|
|
const FormSchema = z
|
|
.object({
|
|
name: z
|
|
.string()
|
|
.min(1, {
|
|
message: t('knowledgeList.namePlaceholder'),
|
|
})
|
|
.trim(),
|
|
parseType: z.number().optional(),
|
|
embd_id: z
|
|
.string()
|
|
.min(1, {
|
|
message: t('knowledgeConfiguration.embeddingModelPlaceholder'),
|
|
})
|
|
.trim(),
|
|
parser_id: z.string().optional(),
|
|
pipeline_id: z.string().optional(),
|
|
language: z.string().optional(),
|
|
})
|
|
.superRefine((data, ctx) => {
|
|
// When parseType === 1, parser_id is required
|
|
if (
|
|
data.parseType === 1 &&
|
|
(!data.parser_id || data.parser_id.trim() === '')
|
|
) {
|
|
ctx.addIssue({
|
|
code: z.ZodIssueCode.custom,
|
|
message: t('knowledgeList.parserRequired'),
|
|
path: ['parser_id'],
|
|
});
|
|
}
|
|
|
|
console.log('form-data', data);
|
|
// When parseType === 1, pipline_id required
|
|
if (data.parseType === 2 && !data.pipeline_id) {
|
|
ctx.addIssue({
|
|
code: z.ZodIssueCode.custom,
|
|
message: t('knowledgeList.dataFlowRequired'),
|
|
path: ['pipeline_id'],
|
|
});
|
|
}
|
|
});
|
|
|
|
const form = useForm<z.infer<typeof FormSchema>>({
|
|
resolver: zodResolver(FormSchema),
|
|
defaultValues: {
|
|
name: '',
|
|
parseType: 1,
|
|
parser_id: '',
|
|
embd_id: '',
|
|
language: 'English',
|
|
},
|
|
});
|
|
|
|
function onSubmit(data: z.infer<typeof FormSchema>) {
|
|
console.log('submit', data);
|
|
onOk?.(data);
|
|
}
|
|
|
|
const parseType = useWatch({
|
|
control: form.control,
|
|
name: 'parseType',
|
|
});
|
|
|
|
useEffect(() => {
|
|
console.log('parseType', parseType);
|
|
if (parseType === 1) {
|
|
form.setValue('pipeline_id', '');
|
|
}
|
|
}, [parseType, form]);
|
|
|
|
return (
|
|
<Form {...form}>
|
|
<form
|
|
onSubmit={form.handleSubmit(onSubmit)}
|
|
className="space-y-6"
|
|
id={FormId}
|
|
>
|
|
<FormField
|
|
control={form.control}
|
|
name="name"
|
|
render={({ field }) => (
|
|
<FormItem>
|
|
<FormLabel>
|
|
<span className="text-destructive mr-1"> *</span>
|
|
{t('knowledgeList.name')}
|
|
</FormLabel>
|
|
<FormControl>
|
|
<Input
|
|
placeholder={t('knowledgeList.namePlaceholder')}
|
|
{...field}
|
|
/>
|
|
</FormControl>
|
|
<FormMessage />
|
|
</FormItem>
|
|
)}
|
|
/>
|
|
|
|
<FormField
|
|
control={form.control}
|
|
name="language"
|
|
render={({ field }) => (
|
|
<FormItem>
|
|
<FormLabel>{t('common.language')}</FormLabel>
|
|
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
|
<FormControl>
|
|
<SelectTrigger>
|
|
<SelectValue
|
|
placeholder={t('common.languagePlaceholder')}
|
|
/>
|
|
</SelectTrigger>
|
|
</FormControl>
|
|
<SelectContent>
|
|
{languageOptions.map((option) => (
|
|
<SelectItem key={option.value} value={option.value}>
|
|
{option.label}
|
|
</SelectItem>
|
|
))}
|
|
</SelectContent>
|
|
</Select>
|
|
<FormMessage />
|
|
</FormItem>
|
|
)}
|
|
/>
|
|
|
|
<EmbeddingModelItem line={2} isEdit={false} />
|
|
<ParseTypeItem />
|
|
{parseType === 1 && <ChunkMethodItem></ChunkMethodItem>}
|
|
{parseType === 2 && (
|
|
<DataFlowSelect
|
|
isMult={false}
|
|
showToDataPipeline={true}
|
|
formFieldName="pipeline_id"
|
|
layout={FormLayout.Vertical}
|
|
/>
|
|
)}
|
|
</form>
|
|
</Form>
|
|
);
|
|
}
|
|
|
|
export function DatasetCreatingDialog({
|
|
hideModal,
|
|
onOk,
|
|
loading,
|
|
}: IModalProps<any>) {
|
|
const { t } = useTranslation();
|
|
|
|
return (
|
|
<Dialog open onOpenChange={hideModal}>
|
|
<DialogContent className="sm:max-w-[425px] focus-visible:!outline-none flex flex-col">
|
|
<DialogHeader>
|
|
<DialogTitle>{t('knowledgeList.createKnowledgeBase')}</DialogTitle>
|
|
</DialogHeader>
|
|
<InputForm onOk={onOk}></InputForm>
|
|
<DialogFooter>
|
|
<ButtonLoading type="submit" form={FormId} loading={loading}>
|
|
{t('common.save')}
|
|
</ButtonLoading>
|
|
</DialogFooter>
|
|
</DialogContent>
|
|
</Dialog>
|
|
);
|
|
}
|