fix github account login issue (#132)

rag/nlp/__init__.py

@@ -76,6 +76,7 @@ def is_english(texts):
 
 def tokenize(d, t, eng):
     d["content_with_weight"] = t
+    t = re.sub(r"</?(table|td|caption|tr|th)( [^<>]{0,12})?>", " ", t)
     if eng:
         t = re.sub(r"([a-z])-([a-z])", r"\1\2", t)
         d["content_ltks"] = " ".join([stemmer.stem(w)

rag/nlp/query.py

@@ -29,7 +29,7 @@ class EsQueryer:
         for t in arr:
             if not re.match(r"[a-zA-Z]+$", t):
                 e += 1
-        return e * 1. / len(arr) >= 0.8
+        return e * 1. / len(arr) >= 0.7
 
     @staticmethod
     def rmWWW(txt):