mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-12-25 08:06:48 +08:00
Feature/mineru improvements (#11938)
我已在下面的评论中用中文重复说明。 ### What problem does this PR solve? ## Summary This PR enhances the MinerU document parser with additional configuration options, giving users more control over PDF parsing behavior and improving support for multilingual documents. ## Changes ### Backend (`deepdoc/parser/mineru_parser.py`) - Added configurable parsing options: - **Parse Method**: `auto`, `txt`, or `ocr` — allows users to choose the extraction strategy - **Formula Recognition**: Toggle for enabling/disabling formula extraction (useful to disable for Cyrillic documents where it may cause issues) - **Table Recognition**: Toggle for enabling/disabling table extraction - Added language code mapping (`LANGUAGE_TO_MINERU_MAP`) to translate RAGFlow language settings to MinerU-compatible language codes for better OCR accuracy - Improved parser configuration handling to pass these options through the processing pipeline ### Frontend (`web/`) - Created new `MinerUOptionsFormField` component that conditionally renders when MinerU is selected as the layout recognition engine - Added UI controls for: - Parse method selection (dropdown) - Formula recognition toggle (switch) - Table recognition toggle (switch) - Added i18n translations for English and Chinese - Integrated the options into both the dataset creation dialog and dataset settings page ### Integration - Updated `rag/app/naive.py` to forward MinerU options to the parser - Updated task service to handle the new configuration parameters ## Why MinerU is a powerful document parser, but the default settings don't work well for all document types. This PR allows users to: 1. Choose the best parsing method for their documents 2. Disable formula recognition for Cyrillic/non-Latin scripts where it causes issues 3. Control table extraction based on document needs 4. Benefit from automatic language detection for better OCR results ## Testing - [x] Tested MinerU parsing with different parse methods - [x] Verified UI renders correctly when MinerU is selected/deselected - [x] Confirmed settings persist correctly in dataset configuration ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [x] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): --------- Co-authored-by: user210 <user210@rt> Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>
This commit is contained in:
97
web/src/components/mineru-options-form-field.tsx
Normal file
97
web/src/components/mineru-options-form-field.tsx
Normal file
@ -0,0 +1,97 @@
|
||||
import { RAGFlowFormItem } from '@/components/ragflow-form';
|
||||
import { RAGFlowSelect } from '@/components/ui/select';
|
||||
import { Switch } from '@/components/ui/switch';
|
||||
import { LLMFactory } from '@/constants/llm';
|
||||
import { buildOptions } from '@/utils/form';
|
||||
import { useFormContext, useWatch } from 'react-hook-form';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
const parseMethodOptions = buildOptions(['auto', 'txt', 'ocr']);
|
||||
|
||||
export function MinerUOptionsFormField() {
|
||||
const form = useFormContext();
|
||||
const { t } = useTranslation();
|
||||
|
||||
const layoutRecognize = useWatch({
|
||||
control: form.control,
|
||||
name: 'parser_config.layout_recognize',
|
||||
});
|
||||
|
||||
// Check if MinerU is selected (the value contains 'MinerU' or matches the factory name)
|
||||
const isMinerUSelected =
|
||||
layoutRecognize?.includes(LLMFactory.MinerU) ||
|
||||
layoutRecognize?.toLowerCase()?.includes('mineru');
|
||||
|
||||
if (!isMinerUSelected) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4 border-l-2 border-primary/30 pl-4 ml-2">
|
||||
<div className="text-sm font-medium text-text-secondary">
|
||||
{t('knowledgeConfiguration.mineruOptions', 'MinerU Options')}
|
||||
</div>
|
||||
|
||||
<RAGFlowFormItem
|
||||
name="parser_config.mineru_parse_method"
|
||||
label={t('knowledgeConfiguration.mineruParseMethod', 'Parse Method')}
|
||||
tooltip={t(
|
||||
'knowledgeConfiguration.mineruParseMethodTip',
|
||||
'Method for parsing PDF: auto (automatic detection), txt (text extraction), ocr (optical character recognition)',
|
||||
)}
|
||||
horizontal={true}
|
||||
>
|
||||
{(field) => (
|
||||
<RAGFlowSelect
|
||||
value={field.value || 'auto'}
|
||||
onChange={field.onChange}
|
||||
options={parseMethodOptions}
|
||||
placeholder={t('common.selectPlaceholder', 'Select value')}
|
||||
/>
|
||||
)}
|
||||
</RAGFlowFormItem>
|
||||
|
||||
<RAGFlowFormItem
|
||||
name="parser_config.mineru_formula_enable"
|
||||
label={t(
|
||||
'knowledgeConfiguration.mineruFormulaEnable',
|
||||
'Formula Recognition',
|
||||
)}
|
||||
tooltip={t(
|
||||
'knowledgeConfiguration.mineruFormulaEnableTip',
|
||||
'Enable formula recognition. Note: This may not work correctly for Cyrillic documents.',
|
||||
)}
|
||||
horizontal={true}
|
||||
labelClassName="!mb-0"
|
||||
>
|
||||
{(field) => (
|
||||
<Switch
|
||||
checked={field.value ?? true}
|
||||
onCheckedChange={field.onChange}
|
||||
/>
|
||||
)}
|
||||
</RAGFlowFormItem>
|
||||
|
||||
<RAGFlowFormItem
|
||||
name="parser_config.mineru_table_enable"
|
||||
label={t(
|
||||
'knowledgeConfiguration.mineruTableEnable',
|
||||
'Table Recognition',
|
||||
)}
|
||||
tooltip={t(
|
||||
'knowledgeConfiguration.mineruTableEnableTip',
|
||||
'Enable table recognition and extraction.',
|
||||
)}
|
||||
horizontal={true}
|
||||
labelClassName="!mb-0"
|
||||
>
|
||||
{(field) => (
|
||||
<Switch
|
||||
checked={field.value ?? true}
|
||||
onCheckedChange={field.onChange}
|
||||
/>
|
||||
)}
|
||||
</RAGFlowFormItem>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user