pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-08 20:42:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

refine pdf parser, add time zone to userinfo (#112)

Browse Source
This commit is contained in:
KevinHuSh
2024-03-08 11:24:24 +08:00
committed by GitHub
parent 63e498ac79
commit 8f86ab9f7f
7 changed files with 28 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
deepdoc/parser/pdf_parser.py
View File

@ -313,9 +313,19 @@ class HuParser:
while i < len(bxs) - 1:
b = bxs[i]
b_ = bxs[i + 1]
if b.get("layoutno", "0") != b_.get("layoutno", "1"):
if b.get("layoutno", "0") != b_.get("layoutno", "1") or b.get("layout_type", "") in ["table", "figure", "equation"]:
i += 1
continue
if abs(self._y_dis(b, b_)) < self.mean_height[bxs[i]["page_number"] - 1] / 3:
# merge
bxs[i]["x1"] = b_["x1"]
bxs[i]["top"] = (b["top"] + b_["top"]) / 2
bxs[i]["bottom"] = (b["bottom"] + b_["bottom"]) / 2
bxs[i]["text"] += b_["text"]
bxs.pop(i + 1)
continue
i += 1
continue
dis_thr = 1
dis = b["x1"] - b_["x0"]
@ -642,9 +652,9 @@ class HuParser:
tk, tv = nearest(tables)
fk, fv = nearest(figures)
if min(tv, fv) > 2000:
i += 1
continue
#if min(tv, fv) > 2000:
# i += 1
# continue
if tv < fv:
tables[tk].insert(0, c)
logging.debug(
@ -711,12 +721,7 @@ class HuParser:
# crop figure out and add caption
for k, bxs in figures.items():
txt = "\n".join(
[b["text"] for b in bxs
if not re.match(r"[0-9a-z.\+%-]", b["text"].strip())
and len(b["text"].strip()) >= 4
]
)
txt = "\n".join([b["text"] for b in bxs])
if not txt:
continue

4
deepdoc/vision/layout_recognizer.py
View File

@ -96,7 +96,7 @@ class LayoutRecognizer(Recognizer):
continue
bxs[i]["layoutno"] = f"{ty}-{ii}"
bxs[i]["layout_type"] = lts_[ii]["type"]
bxs[i]["layout_type"] = lts_[ii]["type"] if lts_[ii]["type"]!="equation" else "figure"
i += 1
for lt in ["footer", "header", "reference", "figure caption",
@ -105,7 +105,7 @@ class LayoutRecognizer(Recognizer):
# add box to figure layouts which has not text box
for i, lt in enumerate(
[lt for lt in lts if lt["type"] == "figure"]):
[lt for lt in lts if lt["type"] in ["figure","equation"]]):
if lt.get("visited"):
continue
lt = deepcopy(lt)

1
deepdoc/vision/ocr.py
View File

@ -21,7 +21,6 @@ from .operators import *
import numpy as np
import onnxruntime as ort
from api.utils.file_utils import get_project_base_directory
from .postprocess import build_post_process
from rag.settings import cron_logger

12
deepdoc/vision/recognizer.py
View File

@ -276,18 +276,18 @@ class Recognizer(object):
def find_overlapped_with_threashold(box, boxes, thr=0.3):
if not boxes:
return
max_overlaped_i, max_overlaped, _max_overlaped = None, thr, 0
max_overlapped_i, max_overlapped, _max_overlapped = None, thr, 0
s, e = 0, len(boxes)
for i in range(s, e):
ov = Recognizer.overlapped_area(box, boxes[i])
_ov = Recognizer.overlapped_area(boxes[i], box)
if (ov, _ov) < (max_overlaped, _max_overlaped):
if (ov, _ov) < (max_overlapped, _max_overlapped):
continue
max_overlaped_i = i
max_overlaped = ov
_max_overlaped = _ov
max_overlapped_i = i
max_overlapped = ov
_max_overlapped = _ov
return max_overlaped_i
return max_overlapped_i
def preprocess(self, image_list):
inputs = []