remove unused codes, seperate layout detection out as a new api. Add new rag methed 'table' (#55)

api/apps/__init__.py

@@ -28,8 +28,6 @@ from api.utils import CustomJSONEncoder
 from flask_session import Session
 from flask_login import LoginManager
 from api.settings import RetCode, SECRET_KEY, stat_logger
-from api.hook import HookManager
-from api.hook.common.parameters import AuthenticationParameters, ClientAuthenticationParameters
 from api.settings import API_VERSION, CLIENT_AUTHENTICATION, SITE_AUTHENTICATION, access_logger
 from api.utils.api_utils import get_json_result, server_error_response
 from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
@@ -96,37 +94,7 @@ client_urls_prefix = [
 ]
 
 
-def client_authentication_before_request():
-    result = HookManager.client_authentication(ClientAuthenticationParameters(
-        request.full_path, request.headers,
-        request.form, request.data, request.json,
-    ))
 
-    if result.code != RetCode.SUCCESS:
-        return get_json_result(result.code, result.message)
-
-
-def site_authentication_before_request():
-    for url_prefix in client_urls_prefix:
-        if request.path.startswith(url_prefix):
-            return
-
-    result = HookManager.site_authentication(AuthenticationParameters(
-        request.headers.get('site_signature'),
-        request.json,
-    ))
-
-    if result.code != RetCode.SUCCESS:
-        return get_json_result(result.code, result.message)
-
-
-@app.before_request
-def authentication_before_request():
-    if CLIENT_AUTHENTICATION:
-        return client_authentication_before_request()
-
-    if SITE_AUTHENTICATION:
-        return site_authentication_before_request()
 
 @login_manager.request_loader
 def load_user(web_request):

api/apps/chunk_app.py

@@ -57,7 +57,7 @@ def list():
         for id in sres.ids:
             d = {
                 "chunk_id": id,
-                "content_ltks": rmSpace(sres.highlight[id]) if question else sres.field[id]["content_ltks"],
+                "content_with_weight": rmSpace(sres.highlight[id]) if question else sres.field[id]["content_with_weight"],
                 "doc_id": sres.field[id]["doc_id"],
                 "docnm_kwd": sres.field[id]["docnm_kwd"],
                 "important_kwd": sres.field[id].get("important_kwd", []),
@@ -134,7 +134,7 @@ def set():
             q, a = rmPrefix(arr[0]), rmPrefix[arr[1]]
             d = beAdoc(d, arr[0], arr[1], not any([huqie.is_chinese(t) for t in q+a]))
 
-        v, c = embd_mdl.encode([doc.name, req["content_ltks"]])
+        v, c = embd_mdl.encode([doc.name, req["content_with_weight"]])
         v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
         d["q_%d_vec" % len(v)] = v.tolist()
         ELASTICSEARCH.upsert([d], search.index_name(tenant_id))
@@ -175,13 +175,13 @@ def rm():
 
 @manager.route('/create', methods=['POST'])
 @login_required
-@validate_request("doc_id", "content_ltks")
+@validate_request("doc_id", "content_with_weight")
 def create():
     req = request.json
     md5 = hashlib.md5()
-    md5.update((req["content_ltks"] + req["doc_id"]).encode("utf-8"))
+    md5.update((req["content_with_weight"] + req["doc_id"]).encode("utf-8"))
     chunck_id = md5.hexdigest()
-    d = {"id": chunck_id, "content_ltks": huqie.qie(req["content_ltks"])}
+    d = {"id": chunck_id, "content_ltks": huqie.qie(req["content_with_weight"])}
     d["content_sm_ltks"] = huqie.qieqie(d["content_ltks"])
     d["important_kwd"] = req.get("important_kwd", [])
     d["important_tks"] = huqie.qie(" ".join(req.get("important_kwd", [])))
@@ -201,7 +201,7 @@ def create():
 
         embd_mdl = TenantLLMService.model_instance(
             tenant_id, LLMType.EMBEDDING.value)
-        v, c = embd_mdl.encode([doc.name, req["content_ltks"]])
+        v, c = embd_mdl.encode([doc.name, req["content_with_weight"]])
         DocumentService.increment_chunk_num(req["doc_id"], doc.kb_id, c, 1, 0)
         v = 0.1 * v[0] + 0.9 * v[1]
         d["q_%d_vec" % len(v)] = v.tolist()

api/apps/conversation_app.py

@@ -175,7 +175,7 @@ def chat(dialog, messages, **kwargs):
     chat_mdl = LLMBundle(dialog.tenant_id, LLMType.CHAT, dialog.llm_id)
     kbinfos = retrievaler.retrieval(question, embd_mdl, dialog.tenant_id, dialog.kb_ids, 1, dialog.top_n, dialog.similarity_threshold,
                         dialog.vector_similarity_weight, top=1024, aggs=False)
-    knowledges = [ck["content_ltks"] for ck in kbinfos["chunks"]]
+    knowledges = [ck["content_with_weight"] for ck in kbinfos["chunks"]]
 
     if not knowledges and prompt_config["empty_response"]:
         return {"answer": prompt_config["empty_response"], "retrieval": kbinfos}

api/apps/document_app.py

@@ -73,6 +73,7 @@ def upload():
             "id": get_uuid(),
             "kb_id": kb.id,
             "parser_id": kb.parser_id,
+            "parser_config": kb.parser_config,
             "created_by": current_user.id,
             "type": filename_type(filename),
             "name": filename,
@@ -108,6 +109,7 @@ def create():
             "id": get_uuid(),
             "kb_id": kb.id,
             "parser_id": kb.parser_id,
+            "parser_config": kb.parser_config,
             "created_by": current_user.id,
             "type": FileType.VIRTUAL,
             "name": req["name"],
@@ -128,8 +130,8 @@ def list():
             data=False, retmsg='Lack of "KB ID"', retcode=RetCode.ARGUMENT_ERROR)
     keywords = request.args.get("keywords", "")
 
-    page_number = request.args.get("page", 1)
-    items_per_page = request.args.get("page_size", 15)
+    page_number = int(request.args.get("page", 1))
+    items_per_page = int(request.args.get("page_size", 15))
     orderby = request.args.get("orderby", "create_time")
     desc = request.args.get("desc", True)
     try:
@@ -214,7 +216,9 @@ def run():
     req = request.json
     try:
         for id in req["doc_ids"]:
-            DocumentService.update_by_id(id, {"run": str(req["run"]), "progress": 0})
+            info = {"run": str(req["run"]), "progress": 0}
+            if str(req["run"]) == TaskStatus.RUNNING.value:info["progress_msg"] = ""
+            DocumentService.update_by_id(id, info)
             if str(req["run"]) == TaskStatus.CANCEL.value:
                 tenant_id = DocumentService.get_tenant_id(id)
                 if not tenant_id:

api/apps/kb_app.py

@@ -29,7 +29,7 @@ from api.utils.api_utils import get_json_result
 
 @manager.route('/create', methods=['post'])
 @login_required
-@validate_request("name", "description", "permission", "parser_id")
+@validate_request("name")
 def create():
     req = request.json
     req["name"] = req["name"].strip()