Add get_txt function (#2639)

### What problem does this PR solve?

Add get_txt function to reduce duplicate code

### Type of change

- [x] Refactoring

---------

Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>
This commit is contained in:
yqkcn
2024-09-29 10:29:56 +08:00
committed by GitHub
parent 57237634f1
commit aea553c3a8
8 changed files with 52 additions and 88 deletions

View File

@ -17,6 +17,7 @@ from io import BytesIO
from docx import Document
from api.db import ParserType
from deepdoc.parser.utils import get_txt
from rag.nlp import bullets_category, is_english, tokenize, remove_contents_table, hierarchical_merge, \
make_colon_as_title, add_positions, tokenize_chunks, find_codec, docx_question_level
from rag.nlp import rag_tokenizer
@ -165,17 +166,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
elif re.search(r"\.txt$", filename, re.IGNORECASE):
callback(0.1, "Start to parse.")
txt = ""
if binary:
encoding = find_codec(binary)
txt = binary.decode(encoding, errors="ignore")
else:
with open(filename, "r") as f:
while True:
l = f.readline()
if not l:
break
txt += l
txt = get_txt(filename, binary)
sections = txt.split("\n")
sections = [l for l in sections if l]
callback(0.8, "Finish parsing.")