Rework logging (#3358)

Unified all log files into one.

### What problem does this PR solve?

Unified all log files into one.

### Type of change

- [x] Refactoring

rag/nlp/rag_tokenizer.py

@@ -22,10 +22,10 @@ import re
 import string
 import sys
 from hanziconv import HanziConv
-from huggingface_hub import snapshot_download
 from nltk import word_tokenize
 from nltk.stem import PorterStemmer, WordNetLemmatizer
 from api.utils.file_utils import get_project_base_directory
+from api.utils.log_utils import logger
 
 
 class RagTokenizer:
@@ -36,7 +36,7 @@ class RagTokenizer:
         return str(("DD" + (line[::-1].lower())).encode("utf-8"))[2:-1]
 
     def loadDict_(self, fnm):
-        print("[HUQIE]:Build trie", fnm, file=sys.stderr)
+        logger.info(f"[HUQIE]:Build trie {fnm}")
         try:
             of = open(fnm, "r", encoding='utf-8')
             while True:
@@ -52,8 +52,9 @@ class RagTokenizer:
                 self.trie_[self.rkey_(line[0])] = 1
             self.trie_.save(fnm + ".trie")
             of.close()
-        except Exception as e:
-            print("[HUQIE]:Faild to build trie, ", fnm, e, file=sys.stderr)
+        except Exception:
+            logger.exception(f"[HUQIE]:Build trie {fnm} failed")
+
 
     def __init__(self, debug=False):
         self.DEBUG = debug
@@ -68,8 +69,8 @@ class RagTokenizer:
         try:
             self.trie_ = datrie.Trie.load(self.DIR_ + ".txt.trie")
             return
-        except Exception as e:
-            print("[HUQIE]:Build default trie", file=sys.stderr)
+        except Exception:
+            logger.exception("[HUQIE]:Build default trie")
             self.trie_ = datrie.Trie(string.printable)
 
         self.loadDict_(self.DIR_ + ".txt")
@@ -78,7 +79,7 @@ class RagTokenizer:
         try:
             self.trie_ = datrie.Trie.load(fnm + ".trie")
             return
-        except Exception as e:
+        except Exception:
             self.trie_ = datrie.Trie(string.printable)
         self.loadDict_(fnm)
 
@@ -173,8 +174,7 @@ class RagTokenizer:
             tks.append(tk)
         F /= len(tks)
         L /= len(tks)
-        if self.DEBUG:
-            print("[SC]", tks, len(tks), L, F, B / len(tks) + L + F)
+        logger.debug("[SC] {} {} {} {} {}".format(tks, len(tks), L, F, B / len(tks) + L + F))
         return tks, B / len(tks) + L + F
 
     def sortTks_(self, tkslist):
@@ -278,8 +278,8 @@ class RagTokenizer:
             tks, s = self.maxForward_(L)
             tks1, s1 = self.maxBackward_(L)
             if self.DEBUG:
-                print("[FW]", tks, s)
-                print("[BW]", tks1, s1)
+                logger.debug("[FW] {} {}".format(tks, s))
+                logger.debug("[BW] {} {}".format(tks1, s1))
 
             i, j, _i, _j = 0, 0, 0, 0
             same = 0
@@ -326,8 +326,7 @@ class RagTokenizer:
                 res.append(" ".join(self.sortTks_(tkslist)[0][0]))
 
         res = " ".join(self.english_normalize_(res))
-        if self.DEBUG:
-            print("[TKS]", self.merge_(res))
+        logger.debug("[TKS] {}".format(self.merge_(res)))
         return self.merge_(res)
 
     def fine_grained_tokenize(self, tks):
@@ -418,30 +417,30 @@ if __name__ == '__main__':
     # huqie.addUserDict("/tmp/tmp.new.tks.dict")
     tks = tknzr.tokenize(
         "哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     tks = tknzr.tokenize(
         "公开征求意见稿提出，境外投资者可使用自有人民币或外汇投资。使用外汇投资的，可通过债券持有人在香港人民币业务清算行及香港地区经批准可进入境内银行间外汇市场进行交易的境外人民币业务参加行（以下统称香港结算行）办理外汇资金兑换。香港结算行由此所产生的头寸可到境内银行间外汇市场平盘。使用外汇投资的，在其投资的债券到期或卖出后，原则上应兑换回外汇。")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     tks = tknzr.tokenize(
         "多校划片就是一个小区对应多个小学初中，让买了学区房的家庭也不确定到底能上哪个学校。目的是通过这种方式为学区房降温，把就近入学落到实处。南京市长江大桥")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     tks = tknzr.tokenize(
         "实际上当时他们已经将业务中心偏移到安全部门和针对政府企业的部门 Scripts are compiled and cached aaaaaaaaa")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     tks = tknzr.tokenize("虽然我不怎么玩")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     tks = tknzr.tokenize("蓝月亮如何在外资夹击中生存,那是全宇宙最有意思的")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     tks = tknzr.tokenize(
         "涡轮增压发动机num最大功率,不像别的共享买车锁电子化的手段,我们接过来是否有意义,黄黄爱美食,不过，今天阿奇要讲到的这家农贸市场，说实话，还真蛮有特色的！不仅环境好，还打出了")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     tks = tknzr.tokenize("这周日你去吗？这周日你有空吗？")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     tks = tknzr.tokenize("Unity3D开发经验 测试开发工程师 c++双11双11 985 211 ")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     tks = tknzr.tokenize(
         "数据分析项目经理|数据分析挖掘|数据分析方向|商品数据分析|搜索数据分析 sql python hive tableau Cocos2d-")
-    print(tknzr.fine_grained_tokenize(tks))
+    logger.info(tknzr.fine_grained_tokenize(tks))
     if len(sys.argv) < 2:
         sys.exit()
     tknzr.DEBUG = False
@@ -451,5 +450,5 @@ if __name__ == '__main__':
         line = of.readline()
         if not line:
             break
-        print(tknzr.tokenize(line))
+        logger.info(tknzr.tokenize(line))
     of.close()