pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-08 20:42:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

add rerank model (#969)

Browse Source 
### What problem does this PR solve?

feat: add rerank models to the project #724 #162

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
KevinHuSh
2024-05-29 16:50:02 +08:00
committed by GitHub
parent e1f0644deb
commit 614defec21
17 changed files with 437 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
rag/nlp/rag_tokenizer.py
View File

@ -241,11 +241,14 @@ class RagTokenizer:
return self.score_(res[::-1])
def english_normalize_(self, tks):
return [self.stemmer.stem(self.lemmatizer.lemmatize(t)) if re.match(r"[a-zA-Z_-]+$", t) else t for t in tks]
def tokenize(self, line):
line = self._strQ2B(line).lower()
line = self._tradi2simp(line)
zh_num = len([1 for c in line if is_chinese(c)])
if zh_num < len(line) * 0.2:
if zh_num == 0:
return " ".join([self.stemmer.stem(self.lemmatizer.lemmatize(t)) for t in word_tokenize(line)])
arr = re.split(self.SPLIT_CHAR, line)
@ -293,7 +296,7 @@ class RagTokenizer:
i = e + 1
res = " ".join(res)
res = " ".join(self.english_normalize_(res))
if self.DEBUG:
print("[TKS]", self.merge_(res))
return self.merge_(res)
@ -336,7 +339,7 @@ class RagTokenizer:
res.append(stk)
return " ".join(res)
return " ".join(self.english_normalize_(res))
def is_chinese(s):