mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-12-18 19:46:44 +08:00
ad6f7fd4b0
### What problem does this PR solve? Fix pipeline ignore MinerU backend config and vllm module is missing. #11944, #11947. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
87 lines
3.6 KiB
Python
87 lines
3.6 KiB
Python
#
|
|
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
import json
|
|
import logging
|
|
import os
|
|
from typing import Any, Optional, Tuple
|
|
|
|
from deepdoc.parser.mineru_parser import MinerUParser
|
|
|
|
|
|
class Base:
|
|
def __init__(self, key: str | dict, model_name: str, **kwargs):
|
|
self.model_name = model_name
|
|
|
|
def parse_pdf(self, filepath: str, binary=None, **kwargs) -> Tuple[Any, Any]:
|
|
raise NotImplementedError("Please implement parse_pdf!")
|
|
|
|
|
|
class MinerUOcrModel(Base, MinerUParser):
|
|
_FACTORY_NAME = "MinerU"
|
|
|
|
def __init__(self, key: str | dict, model_name: str, **kwargs):
|
|
Base.__init__(self, key, model_name, **kwargs)
|
|
raw_config = {}
|
|
if key:
|
|
try:
|
|
raw_config = json.loads(key)
|
|
except Exception:
|
|
raw_config = {}
|
|
|
|
# nested {"api_key": {...}} from UI
|
|
# flat {"MINERU_*": "..."} payload auto-provisioned from env vars
|
|
config = raw_config.get("api_key", raw_config)
|
|
if not isinstance(config, dict):
|
|
config = {}
|
|
|
|
def _resolve_config(key: str, env_key: str, default=""):
|
|
# lower-case keys (UI), upper-case MINERU_* (env auto-provision), env vars
|
|
return config.get(key, config.get(env_key, os.environ.get(env_key, default)))
|
|
|
|
self.mineru_api = _resolve_config("mineru_apiserver", "MINERU_APISERVER", "")
|
|
self.mineru_output_dir = _resolve_config("mineru_output_dir", "MINERU_OUTPUT_DIR", "")
|
|
self.mineru_backend = _resolve_config("mineru_backend", "MINERU_BACKEND", "pipeline")
|
|
self.mineru_server_url = _resolve_config("mineru_server_url", "MINERU_SERVER_URL", "")
|
|
self.mineru_delete_output = bool(int(_resolve_config("mineru_delete_output", "MINERU_DELETE_OUTPUT", 1)))
|
|
self.mineru_executable = os.environ.get("MINERU_EXECUTABLE", "mineru")
|
|
|
|
logging.info(f"Parsed MinerU config: {config}")
|
|
|
|
MinerUParser.__init__(self, mineru_path=self.mineru_executable, mineru_api=self.mineru_api, mineru_server_url=self.mineru_server_url)
|
|
|
|
def check_available(self, backend: Optional[str] = None, server_url: Optional[str] = None) -> Tuple[bool, str]:
|
|
backend = backend or self.mineru_backend
|
|
server_url = server_url or self.mineru_server_url
|
|
return self.check_installation(backend=backend, server_url=server_url)
|
|
|
|
def parse_pdf(self, filepath: str, binary=None, callback=None, parse_method: str = "raw", **kwargs):
|
|
ok, reason = self.check_available()
|
|
if not ok:
|
|
raise RuntimeError(f"MinerU not found or server not accessible: {reason}. Please install it via: pip install -U 'mineru[core]'.")
|
|
|
|
sections, tables = MinerUParser.parse_pdf(
|
|
self,
|
|
filepath=filepath,
|
|
binary=binary,
|
|
callback=callback,
|
|
output_dir=self.mineru_output_dir,
|
|
backend=self.mineru_backend,
|
|
server_url=self.mineru_server_url,
|
|
delete_output=self.mineru_delete_output,
|
|
parse_method=parse_method,
|
|
)
|
|
return sections, tables
|