pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-08 20:42:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

refine table parser (#120)

Browse Source
This commit is contained in:
KevinHuSh
2024-03-12 18:56:04 +08:00
committed by GitHub
parent f1f09df901
commit 0feb085c88
6 changed files with 20 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
rag/svr/task_executor.py
View File

@ -128,8 +128,6 @@ def build(row):
return
callback(msg="Finished slicing files(%d). Start to embedding the content."%len(cks))
docs = []
doc = {
"doc_id": row["doc_id"],
@ -179,8 +177,8 @@ def embedding(docs, mdl, parser_config={}, callback=None):
tk_count += c
cnts_ = np.array([])
for i in range(0, len(cnts), 32):
vts, c = mdl.encode(cnts[i: i+32])
for i in range(0, len(cnts), 8):
vts, c = mdl.encode(cnts[i: i+8])
if len(cnts_) == 0: cnts_ = vts
else: cnts_ = np.concatenate((cnts_, vts), axis=0)
tk_count += c
@ -226,6 +224,7 @@ def main(comm, mod):
continue
# TODO: exception handler
## set_progress(r["did"], -1, "ERROR: ")
callback(msg="Finished slicing files(%d). Start to embedding the content."%len(cks))
try:
tk_count = embedding(cks, embd_mdl, r["parser_config"], callback)
except Exception as e: