support snapshot download from local (#153)

* support snapshot download from local * let snapshot download from local
2025-12-08 12:32:30 +08:00 · 2024-03-27 09:53:42 +08:00
parent da21320b88
commit 979b3a5b4b
12 changed files with 109 additions and 24 deletions
--- a/rag/svr/task_executor.py
+++ b/rag/svr/task_executor.py
@ -172,7 +172,7 @@ def init_kb(row):
 def embedding(docs, mdl, parser_config={}, callback=None):
    batch_size = 32
    tts, cnts = [rmSpace(d["title_tks"]) for d in docs if d.get("title_tks")], [
-        d["content_with_weight"] for d in docs]
+        re.sub(r"</?(table|td|caption|tr|th)( [^<>]{0,12})?>", " ", d["content_with_weight"]) for d in docs]
    tk_count = 0
    if len(tts) == len(cnts):
        tts_ = np.array([])