mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-12-23 06:46:40 +08:00
Feat: enhance Excel image extraction with vision-based descriptions (#12054)
### What problem does this PR solve? issue: [#11618](https://github.com/infiniflow/ragflow/issues/11618) change: enhance Excel image extraction with vision-based descriptions ### Type of change - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
@ -18,6 +18,7 @@ from io import BytesIO
|
||||
|
||||
import pandas as pd
|
||||
from openpyxl import Workbook, load_workbook
|
||||
from PIL import Image
|
||||
|
||||
from rag.nlp import find_codec
|
||||
|
||||
@ -109,6 +110,52 @@ class RAGFlowExcelParser:
|
||||
ws.cell(row=row_num, column=col_num, value=value)
|
||||
return wb
|
||||
|
||||
@staticmethod
|
||||
def _extract_images_from_worksheet(ws, sheetname=None):
|
||||
"""
|
||||
Extract images from a worksheet and enrich them with vision-based descriptions.
|
||||
|
||||
Returns: List[dict]
|
||||
"""
|
||||
images = getattr(ws, "_images", [])
|
||||
if not images:
|
||||
return []
|
||||
|
||||
raw_items = []
|
||||
|
||||
for img in images:
|
||||
try:
|
||||
img_bytes = img._data()
|
||||
pil_img = Image.open(BytesIO(img_bytes)).convert("RGB")
|
||||
|
||||
anchor = img.anchor
|
||||
if hasattr(anchor, "_from") and hasattr(anchor, "_to"):
|
||||
r1, c1 = anchor._from.row + 1, anchor._from.col + 1
|
||||
r2, c2 = anchor._to.row + 1, anchor._to.col + 1
|
||||
if r1 == r2 and c1 == c2:
|
||||
span = "single_cell"
|
||||
else:
|
||||
span = "multi_cell"
|
||||
else:
|
||||
r1, c1 = anchor._from.row + 1, anchor._from.col + 1
|
||||
r2, c2 = r1, c1
|
||||
span = "single_cell"
|
||||
|
||||
item = {
|
||||
"sheet": sheetname or ws.title,
|
||||
"image": pil_img,
|
||||
"image_description": "",
|
||||
"row_from": r1,
|
||||
"col_from": c1,
|
||||
"row_to": r2,
|
||||
"col_to": c2,
|
||||
"span_type": span,
|
||||
}
|
||||
raw_items.append(item)
|
||||
except Exception:
|
||||
continue
|
||||
return raw_items
|
||||
|
||||
def html(self, fnm, chunk_rows=256):
|
||||
from html import escape
|
||||
|
||||
|
||||
@ -55,6 +55,31 @@ def vision_figure_parser_docx_wrapper(sections, tbls, callback=None,**kwargs):
|
||||
callback(0.8, f"Visual model error: {e}. Skipping figure parsing enhancement.")
|
||||
return tbls
|
||||
|
||||
def vision_figure_parser_figure_xlsx_wrapper(images,callback=None, **kwargs):
|
||||
tbls = []
|
||||
if not images:
|
||||
return []
|
||||
try:
|
||||
vision_model = LLMBundle(kwargs["tenant_id"], LLMType.IMAGE2TEXT)
|
||||
callback(0.2, "Visual model detected. Attempting to enhance Excel image extraction...")
|
||||
except Exception:
|
||||
vision_model = None
|
||||
if vision_model:
|
||||
figures_data = [((
|
||||
img["image"], # Image.Image
|
||||
[img["image_description"]] # description list (must be list)
|
||||
),
|
||||
[
|
||||
(0, 0, 0, 0, 0) # dummy position
|
||||
]) for img in images]
|
||||
try:
|
||||
parser = VisionFigureParser(vision_model=vision_model, figures_data=figures_data, **kwargs)
|
||||
callback(0.22, "Parsing images...")
|
||||
boosted_figures = parser(callback=callback)
|
||||
tbls.extend(boosted_figures)
|
||||
except Exception as e:
|
||||
callback(0.25, f"Excel visual model error: {e}. Skipping vision enhancement.")
|
||||
return tbls
|
||||
|
||||
def vision_figure_parser_pdf_wrapper(tbls, callback=None, **kwargs):
|
||||
if not tbls:
|
||||
|
||||
Reference in New Issue
Block a user