From 08d563777043c907b7bd1edc75babb57c3fd3b4a Mon Sep 17 00:00:00 2001
From: lidp <43509927+guoyuhao2330@users.noreply.github.com>
Date: Wed, 25 Sep 2024 10:30:49 +0800
Subject: [PATCH] Fix tokenizer bug (#2573)

### What problem does this PR solve?

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
---
 rag/nlp/rag_tokenizer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rag/nlp/rag_tokenizer.py b/rag/nlp/rag_tokenizer.py
index 11881162d..eee013e61 100644
--- a/rag/nlp/rag_tokenizer.py
+++ b/rag/nlp/rag_tokenizer.py
@@ -64,7 +64,7 @@ class RagTokenizer:
         self.stemmer = PorterStemmer()
         self.lemmatizer = WordNetLemmatizer()
 
-        self.SPLIT_CHAR = r"([ ,\.<>/?;'\[\]\\`!@#$%^&*\(\)\{\}\|_+=《》，。？、；‘’：“”【】~！￥%……（）——-]+|[a-z\.-]+|[0-9,\.-]+)"
+        self.SPLIT_CHAR = r"([ ,\.<>/?;:'\[\]\\`!@#$%^&*\(\)\{\}\|_+=《》，。？、；‘’：“”【】~！￥%……（）——-]+|[a-z\.-]+|[0-9,\.-]+)"
         try:
             self.trie_ = datrie.Trie.load(self.DIR_ + ".txt.trie")
             return