fix LIGHTEN issue (#2806)

### What problem does this PR solve?


### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
Kevin Hu
2024-10-11 15:01:27 +08:00
committed by GitHub
parent 3f065c75da
commit 2d1c83da59
3 changed files with 2 additions and 13 deletions

View File

@ -501,17 +501,6 @@ def naive_merge(sections, chunk_token_num=128, delimiter="\n。"):
for sec, pos in sections:
add_chunk(sec, pos)
continue
s, e = 0, 1
while e < len(sec):
if sec[e] in delimiter:
add_chunk(sec[s: e + 1], pos)
s = e + 1
e = s + 1
else:
e += 1
if s < e:
add_chunk(sec[s: e], pos)
return cks