add alot of api (#23)

* clean rust version project

* clean rust version project

* build python version rag-flow

* add alot of api

web_server/apps/document_app.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import base64
 import pathlib
 
 from elasticsearch_dsl import Q
@@ -195,11 +196,15 @@ def rm():
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
             return get_data_error_result(retmsg="Document not found!")
+        if not ELASTICSEARCH.deleteByQuery(Q("match", doc_id=doc.id), idxnm=search.index_name(doc.kb_id)):
+            return get_json_result(data=False, retmsg='Remove from ES failure"', retcode=RetCode.SERVER_ERROR)
+
+        DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num*-1, doc.chunk_num*-1, 0)
         if not DocumentService.delete_by_id(req["doc_id"]):
             return get_data_error_result(
                 retmsg="Database error (Document removal)!")
-        e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
-        MINIO.rm(kb.id, doc.location)
+
+        MINIO.rm(doc.kb_id, doc.location)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
@@ -233,3 +238,43 @@ def rename():
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
+
+
+@manager.route('/get', methods=['GET'])
+@login_required
+def get():
+    doc_id = request.args["doc_id"]
+    try:
+        e, doc = DocumentService.get_by_id(doc_id)
+        if not e:
+            return get_data_error_result(retmsg="Document not found!")
+
+        blob = MINIO.get(doc.kb_id, doc.location)
+        return get_json_result(data={"base64": base64.b64decode(blob)})
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route('/change_parser', methods=['POST'])
+@login_required
+@validate_request("doc_id", "parser_id")
+def change_parser():
+    req = request.json
+    try:
+        e, doc = DocumentService.get_by_id(req["doc_id"])
+        if not e:
+            return get_data_error_result(retmsg="Document not found!")
+        if doc.parser_id.lower() == req["parser_id"].lower():
+            return get_json_result(data=True)
+
+        e = DocumentService.update_by_id(doc.id, {"parser_id": req["parser_id"], "progress":0, "progress_msg": ""})
+        if not e:
+            return get_data_error_result(retmsg="Document not found!")
+        e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num*-1, doc.chunk_num*-1, doc.process_duation*-1)
+        if not e:
+            return get_data_error_result(retmsg="Document not found!")
+
+        return get_json_result(data=True)
+    except Exception as e:
+        return server_error_response(e)
+

web_server/apps/kb_app.py

@@ -29,7 +29,7 @@ from web_server.utils.api_utils import get_json_result
 
 @manager.route('/create', methods=['post'])
 @login_required
-@validate_request("name", "description", "permission", "embd_id", "parser_id")
+@validate_request("name", "description", "permission", "parser_id")
 def create():
     req = request.json
     req["name"] = req["name"].strip()
@@ -46,7 +46,7 @@ def create():
 
 @manager.route('/update', methods=['post'])
 @login_required
-@validate_request("kb_id", "name", "description", "permission", "embd_id", "parser_id")
+@validate_request("kb_id", "name", "description", "permission", "parser_id")
 def update():
     req = request.json
     req["name"] = req["name"].strip()
@@ -72,6 +72,18 @@ def update():
         return server_error_response(e)
 
 
+@manager.route('/detail', methods=['GET'])
+@login_required
+def detail():
+    kb_id = request.args["kb_id"]
+    try:
+        kb = KnowledgebaseService.get_detail(kb_id)
+        if not kb: return get_data_error_result(retmsg="Can't find this knowledgebase!")
+        return get_json_result(data=kb)
+    except Exception as e:
+        return server_error_response(e)
+
+
 @manager.route('/list', methods=['GET'])
 @login_required
 def list():

web_server/apps/llm_app.py

@@ -0,0 +1,95 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from flask import request
+from flask_login import login_required, current_user
+
+from web_server.db.services import duplicate_name
+from web_server.db.services.llm_service import LLMFactoriesService, TenantLLMService, LLMService
+from web_server.db.services.user_service import TenantService, UserTenantService
+from web_server.utils.api_utils import server_error_response, get_data_error_result, validate_request
+from web_server.utils import get_uuid, get_format_time
+from web_server.db import StatusEnum, UserTenantRole
+from web_server.db.services.kb_service import KnowledgebaseService
+from web_server.db.db_models import Knowledgebase, TenantLLM
+from web_server.settings import stat_logger, RetCode
+from web_server.utils.api_utils import get_json_result
+
+
+@manager.route('/factories', methods=['GET'])
+@login_required
+def factories():
+    try:
+        fac = LLMFactoriesService.get_all()
+        return get_json_result(data=fac.to_json())
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route('/set_api_key', methods=['POST'])
+@login_required
+@validate_request("llm_factory", "api_key")
+def set_api_key():
+    req = request.json
+    llm = {
+        "tenant_id": current_user.id,
+        "llm_factory": req["llm_factory"],
+        "api_key": req["api_key"]
+    }
+    # TODO: Test api_key
+    for n in ["model_type", "llm_name"]:
+        if n in req: llm[n] = req[n]
+
+    TenantLLM.insert(**llm).on_conflict("replace").execute()
+    return get_json_result(data=True)
+
+
+@manager.route('/my_llms', methods=['GET'])
+@login_required
+def my_llms():
+    try:
+        objs = TenantLLMService.query(tenant_id=current_user.id)
+        objs = [o.to_dict() for o in objs]
+        for o in objs: del o["api_key"]
+        return get_json_result(data=objs)
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route('/list', methods=['GET'])
+@login_required
+def list():
+    try:
+        objs = TenantLLMService.query(tenant_id=current_user.id)
+        objs = [o.to_dict() for o in objs if o.api_key]
+        fct = {}
+        for o in objs:
+            if o["llm_factory"] not in fct: fct[o["llm_factory"]] = []
+            if o["llm_name"]: fct[o["llm_factory"]].append(o["llm_name"])
+
+        llms = LLMService.get_all()
+        llms = [m.to_dict() for m in llms if m.status == StatusEnum.VALID.value]
+        for m in llms:
+            m["available"] = False
+            if m["fid"] in fct and (not fct[m["fid"]] or m["llm_name"] in fct[m["fid"]]):
+                m["available"] = True
+        res = {}
+        for m in llms:
+            if m["fid"] not in res: res[m["fid"]] = []
+            res[m["fid"]].append(m)
+
+        return get_json_result(data=res)
+    except Exception as e:
+        return server_error_response(e)

web_server/apps/user_app.py

@@ -16,9 +16,12 @@
 from flask import request, session, redirect, url_for
 from werkzeug.security import generate_password_hash, check_password_hash
 from flask_login import login_required, current_user, login_user, logout_user
+
+from web_server.db.db_models import TenantLLM
+from web_server.db.services.llm_service import TenantLLMService
 from web_server.utils.api_utils import server_error_response, validate_request
 from web_server.utils import get_uuid, get_format_time, decrypt, download_img
-from web_server.db import UserTenantRole
+from web_server.db import UserTenantRole, LLMType
 from web_server.settings import RetCode, GITHUB_OAUTH, CHAT_MDL, EMBEDDING_MDL, ASR_MDL, IMAGE2TEXT_MDL, PARSERS
 from web_server.db.services.user_service import UserService, TenantService, UserTenantService
 from web_server.settings import stat_logger
@@ -47,8 +50,9 @@ def login():
             avatar = download_img(userinfo["avatar_url"])
         except Exception as e:
             stat_logger.exception(e)
+        user_id = get_uuid()
         try:
-            users = user_register({
+            users = user_register(user_id, {
                 "access_token": session["access_token"],
                 "email": userinfo["email"],
                 "avatar": avatar,
@@ -63,6 +67,7 @@ def login():
             login_user(user)
             return cors_reponse(data=user.to_json(), auth=user.get_id(), retmsg="Welcome back!")
         except Exception as e:
+            rollback_user_registration(user_id)
             stat_logger.exception(e)
             return server_error_response(e)
     elif not request.json:
@@ -162,7 +167,25 @@ def user_info():
     return get_json_result(data=current_user.to_dict())
 
 
-def user_register(user):
+def rollback_user_registration(user_id):
+    try:
+        TenantService.delete_by_id(user_id)
+    except Exception as e:
+        pass
+    try:
+        u = UserTenantService.query(tenant_id=user_id)
+        if u:
+            UserTenantService.delete_by_id(u[0].id)
+    except Exception as e:
+        pass
+    try:
+        TenantLLM.delete().where(TenantLLM.tenant_id==user_id).excute()
+    except Exception as e:
+        pass
+
+      
+def user_register(user_id, user):
+
     user_id = get_uuid()
     user["id"] = user_id
     tenant = {
@@ -180,10 +203,12 @@ def user_register(user):
         "invited_by": user_id,
         "role": UserTenantRole.OWNER
     }
+    tenant_llm = {"tenant_id": user_id, "llm_factory": "OpenAI", "api_key": "infiniflow API Key"}
 
     if not UserService.save(**user):return
     TenantService.save(**tenant)
     UserTenantService.save(**usr_tenant)
+    TenantLLMService.save(**tenant_llm)
     return UserService.query(email=user["email"])
 
 
@@ -203,14 +228,17 @@ def user_add():
         "last_login_time": get_format_time(),
         "is_superuser": False,
     }
+
+    user_id = get_uuid()
     try:
-        users = user_register(user_dict)
+        users = user_register(user_id, user_dict)
         if not users: raise Exception('Register user failure.')
         if len(users) > 1: raise Exception('Same E-mail exist!')
         user = users[0]
         login_user(user)
         return cors_reponse(data=user.to_json(), auth=user.get_id(), retmsg="Welcome aboard!")
     except Exception as e:
+        rollback_user_registration(user_id)
         stat_logger.exception(e)
         return get_json_result(data=False, retmsg='User registration failure!', retcode=RetCode.EXCEPTION_ERROR)
 
@@ -220,7 +248,7 @@ def user_add():
 @login_required
 def tenant_info():
     try:
-        tenants = TenantService.get_by_user_id(current_user.id)
+        tenants = TenantService.get_by_user_id(current_user.id)[0]
         return get_json_result(data=tenants)
     except Exception as e:
         return server_error_response(e)