pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-08 20:42:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

refine manual parser (#140)

Browse Source
This commit is contained in:
KevinHuSh
2024-03-21 18:17:32 +08:00
committed by GitHub
parent f4ec7cfa76
commit 6c6b144de2
11 changed files with 77 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
api/apps/conversation_app.py
View File

@ -118,14 +118,13 @@ def message_fit_in(msg, max_length=4000):
c = count()
if c < max_length: return c, msg
msg = [m for m in msg if m.role in ["system", "user"]]
c = count()
if c < max_length: return c, msg
msg_ = [m for m in msg[:-1] if m.role == "system"]
msg_.append(msg[-1])
msg = msg_
c = count()
if c < max_length: return c, msg
ll = num_tokens_from_string(msg_[0].content)
l = num_tokens_from_string(msg_[-1].content)
if ll / (ll + l) > 0.8: