pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-08 20:42:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update RetCode to common.constants (#10984)

Browse Source 
### What problem does this PR solve?

1. Update RetCode to common.constants
2. Decouple the admin and API modules

### Type of change

- [x] Refactoring

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2025-11-04 15:12:53 +08:00
committed by GitHub
parent 16d2be623c
commit 03038c7d3d
25 changed files with 257 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
admin/server/auth.py
View File

@ -31,9 +31,7 @@ from api.db import ActiveEnum, StatusEnum
from api.utils.crypt import decrypt from api.utils.crypt import decrypt
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.time_utils import current_timestamp, datetime_format, get_format_time from common.time_utils import current_timestamp, datetime_format, get_format_time
from api.utils.api_utils import ( from common.connection_utils import construct_response
construct_response,
)
def setup_auth(login_manager): def setup_auth(login_manager):

41
api/apps/api_app.py
View File

@ -34,6 +34,7 @@ from api.db.services.task_service import queue_tasks, TaskService
from api.db.services.user_service import UserTenantService from api.db.services.user_service import UserTenantService
from api import settings from api import settings
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.contants import RetCode
from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request, \ from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request, \
generate_confirmation_token generate_confirmation_token
@ -145,7 +146,7 @@ def set_conversation():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
try: try:
if objs[0].source == "agent": if objs[0].source == "agent":
e, cvs = UserCanvasService.get_by_id(objs[0].dialog_id) e, cvs = UserCanvasService.get_by_id(objs[0].dialog_id)
@ -186,7 +187,7 @@ def completion():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
req = request.json req = request.json
e, conv = API4ConversationService.get_by_id(req["conversation_id"]) e, conv = API4ConversationService.get_by_id(req["conversation_id"])
if not e: if not e:
@ -352,7 +353,7 @@ def get_conversation(conversation_id):
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
try: try:
e, conv = API4ConversationService.get_by_id(conversation_id) e, conv = API4ConversationService.get_by_id(conversation_id)
@ -362,7 +363,7 @@ def get_conversation(conversation_id):
conv = conv.to_dict() conv = conv.to_dict()
if token != APIToken.query(dialog_id=conv['dialog_id'])[0].token: if token != APIToken.query(dialog_id=conv['dialog_id'])[0].token:
return get_json_result(data=False, message='Authentication error: API key is invalid for this conversation_id!"', return get_json_result(data=False, message='Authentication error: API key is invalid for this conversation_id!"',
code=settings.RetCode.AUTHENTICATION_ERROR) code=RetCode.AUTHENTICATION_ERROR)
for referenct_i in conv['reference']: for referenct_i in conv['reference']:
if referenct_i is None or len(referenct_i) == 0: if referenct_i is None or len(referenct_i) == 0:
@ -383,7 +384,7 @@ def upload():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
kb_name = request.form.get("kb_name").strip() kb_name = request.form.get("kb_name").strip()
tenant_id = objs[0].tenant_id tenant_id = objs[0].tenant_id
@ -399,12 +400,12 @@ def upload():
if 'file' not in request.files: if 'file' not in request.files:
return get_json_result( return get_json_result(
data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR) data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
file = request.files['file'] file = request.files['file']
if file.filename == '': if file.filename == '':
return get_json_result( return get_json_result(
data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR) data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR)
root_folder = FileService.get_root_folder(tenant_id) root_folder = FileService.get_root_folder(tenant_id)
pf_id = root_folder["id"] pf_id = root_folder["id"]
@ -496,17 +497,17 @@ def upload_parse():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
if 'file' not in request.files: if 'file' not in request.files:
return get_json_result( return get_json_result(
data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR) data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist('file') file_objs = request.files.getlist('file')
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == '': if file_obj.filename == '':
return get_json_result( return get_json_result(
data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR) data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR)
doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, objs[0].tenant_id) doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, objs[0].tenant_id)
return get_json_result(data=doc_ids) return get_json_result(data=doc_ids)
@ -519,7 +520,7 @@ def list_chunks():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
req = request.json req = request.json
@ -559,7 +560,7 @@ def get_chunk(chunk_id):
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
try: try:
tenant_id = objs[0].tenant_id tenant_id = objs[0].tenant_id
kb_ids = KnowledgebaseService.get_kb_ids(tenant_id) kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
@ -584,7 +585,7 @@ def list_kb_docs():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
req = request.json req = request.json
tenant_id = objs[0].tenant_id tenant_id = objs[0].tenant_id
@ -637,7 +638,7 @@ def docinfos():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
req = request.json req = request.json
doc_ids = req["doc_ids"] doc_ids = req["doc_ids"]
docs = DocumentService.get_by_ids(doc_ids) docs = DocumentService.get_by_ids(doc_ids)
@ -651,7 +652,7 @@ def document_rm():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
tenant_id = objs[0].tenant_id tenant_id = objs[0].tenant_id
req = request.json req = request.json
@ -703,7 +704,7 @@ def document_rm():
errors += str(e) errors += str(e)
if errors: if errors:
return get_json_result(data=False, message=errors, code=settings.RetCode.SERVER_ERROR) return get_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
return get_json_result(data=True) return get_json_result(data=True)
@ -718,7 +719,7 @@ def completion_faq():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
e, conv = API4ConversationService.get_by_id(req["conversation_id"]) e, conv = API4ConversationService.get_by_id(req["conversation_id"])
if not e: if not e:
@ -857,7 +858,7 @@ def retrieval():
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result( return get_json_result(
data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR) data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
req = request.json req = request.json
kb_ids = req.get("kb_id", []) kb_ids = req.get("kb_id", [])
@ -876,7 +877,7 @@ def retrieval():
if len(embd_nms) != 1: if len(embd_nms) != 1:
return get_json_result( return get_json_result(
data=False, message='Knowledge bases use different embedding models or does not exist."', data=False, message='Knowledge bases use different embedding models or does not exist."',
code=settings.RetCode.AUTHENTICATION_ERROR) code=RetCode.AUTHENTICATION_ERROR)
embd_mdl = LLMBundle(kbs[0].tenant_id, LLMType.EMBEDDING, llm_name=kbs[0].embd_id) embd_mdl = LLMBundle(kbs[0].tenant_id, LLMType.EMBEDDING, llm_name=kbs[0].embd_id)
rerank_mdl = None rerank_mdl = None
@ -895,5 +896,5 @@ def retrieval():
except Exception as e: except Exception as e:
if str(e).find("not_found") > 0: if str(e).find("not_found") > 0:
return get_json_result(data=False, message='No chunk found! Check the chunk status please!', return get_json_result(data=False, message='No chunk found! Check the chunk status please!',
code=settings.RetCode.DATA_ERROR) code=RetCode.DATA_ERROR)
return server_error_response(e) return server_error_response(e)

2
api/apps/canvas_app.py
View File

@ -34,7 +34,7 @@ from api.db.services.pipeline_operation_log_service import PipelineOperationLogS
from api.db.services.task_service import queue_dataflow, CANVAS_DEBUG_DOC_ID, TaskService from api.db.services.task_service import queue_dataflow, CANVAS_DEBUG_DOC_ID, TaskService
from api.db.services.user_service import TenantService from api.db.services.user_service import TenantService
from api.db.services.user_canvas_version import UserCanvasVersionService from api.db.services.user_canvas_version import UserCanvasVersionService
from api.settings import RetCode from common.contants import RetCode
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result
from agent.canvas import Canvas from agent.canvas import Canvas

11
api/apps/chunk_app.py
View File

@ -36,6 +36,7 @@ from rag.nlp import rag_tokenizer, search
from rag.prompts.generator import gen_meta_filter, cross_languages, keyword_extraction from rag.prompts.generator import gen_meta_filter, cross_languages, keyword_extraction
from rag.settings import PAGERANK_FLD from rag.settings import PAGERANK_FLD
from common.string_utils import remove_redundant_spaces from common.string_utils import remove_redundant_spaces
from common.contants import RetCode
@manager.route('/list', methods=['POST']) # noqa: F821 @manager.route('/list', methods=['POST']) # noqa: F821
@ -83,7 +84,7 @@ def list_chunk():
except Exception as e: except Exception as e:
if str(e).find("not_found") > 0: if str(e).find("not_found") > 0:
return get_json_result(data=False, message='No chunk found!', return get_json_result(data=False, message='No chunk found!',
code=settings.RetCode.DATA_ERROR) code=RetCode.DATA_ERROR)
return server_error_response(e) return server_error_response(e)
@ -115,7 +116,7 @@ def get():
except Exception as e: except Exception as e:
if str(e).find("NotFoundError") >= 0: if str(e).find("NotFoundError") >= 0:
return get_json_result(data=False, message='Chunk not found!', return get_json_result(data=False, message='Chunk not found!',
code=settings.RetCode.DATA_ERROR) code=RetCode.DATA_ERROR)
return server_error_response(e) return server_error_response(e)
@ -292,7 +293,7 @@ def retrieval_test():
kb_ids = [kb_ids] kb_ids = [kb_ids]
if not kb_ids: if not kb_ids:
return get_json_result(data=False, message='Please specify dataset firstly.', return get_json_result(data=False, message='Please specify dataset firstly.',
code=settings.RetCode.DATA_ERROR) code=RetCode.DATA_ERROR)
doc_ids = req.get("doc_ids", []) doc_ids = req.get("doc_ids", [])
use_kg = req.get("use_kg", False) use_kg = req.get("use_kg", False)
@ -326,7 +327,7 @@ def retrieval_test():
else: else:
return get_json_result( return get_json_result(
data=False, message='Only owner of knowledgebase authorized for this operation.', data=False, message='Only owner of knowledgebase authorized for this operation.',
code=settings.RetCode.OPERATING_ERROR) code=RetCode.OPERATING_ERROR)
e, kb = KnowledgebaseService.get_by_id(kb_ids[0]) e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
if not e: if not e:
@ -371,7 +372,7 @@ def retrieval_test():
except Exception as e: except Exception as e:
if str(e).find("not_found") > 0: if str(e).find("not_found") > 0:
return get_json_result(data=False, message='No chunk found! Check the chunk status please!', return get_json_result(data=False, message='No chunk found! Check the chunk status please!',
code=settings.RetCode.DATA_ERROR) code=RetCode.DATA_ERROR)
return server_error_response(e) return server_error_response(e)

5
api/apps/connector_app.py
View File

@ -18,12 +18,11 @@ import time
from flask import request from flask import request
from flask_login import login_required, current_user from flask_login import login_required, current_user
from api import settings
from api.db import TaskStatus, InputType from api.db import TaskStatus, InputType
from api.db.services.connector_service import ConnectorService, Connector2KbService, SyncLogsService from api.db.services.connector_service import ConnectorService, Connector2KbService, SyncLogsService
from api.utils.api_utils import get_json_result, validate_request, get_data_error_result from api.utils.api_utils import get_json_result, validate_request, get_data_error_result
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.contants import RetCode
@manager.route("/set", methods=["POST"]) # noqa: F821 @manager.route("/set", methods=["POST"]) # noqa: F821
@login_required @login_required
@ -95,7 +94,7 @@ def link_kb(connector_id):
req = request.json req = request.json
errors = Connector2KbService.link_kb(connector_id, req["kb_ids"], current_user.id) errors = Connector2KbService.link_kb(connector_id, req["kb_ids"], current_user.id)
if errors: if errors:
return get_json_result(data=False, message=errors, code=settings.RetCode.SERVER_ERROR) return get_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
return get_json_result(data=True) return get_json_result(data=True)

8
api/apps/conversation_app.py
View File

@ -19,7 +19,6 @@ import logging
from copy import deepcopy from copy import deepcopy
from flask import Response, request from flask import Response, request
from flask_login import current_user, login_required from flask_login import current_user, login_required
from api import settings
from api.db import LLMType from api.db import LLMType
from api.db.db_models import APIToken from api.db.db_models import APIToken
from api.db.services.conversation_service import ConversationService, structure_answer from api.db.services.conversation_service import ConversationService, structure_answer
@ -31,6 +30,7 @@ from api.db.services.user_service import TenantService, UserTenantService
from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request
from rag.prompts.template import load_prompt from rag.prompts.template import load_prompt
from rag.prompts.generator import chunks_format from rag.prompts.generator import chunks_format
from common.contants import RetCode
@manager.route("/set", methods=["POST"]) # noqa: F821 @manager.route("/set", methods=["POST"]) # noqa: F821
@ -93,7 +93,7 @@ def get():
avatar = dialog[0].icon avatar = dialog[0].icon
break break
else: else:
return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR) return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=RetCode.OPERATING_ERROR)
for ref in conv.reference: for ref in conv.reference:
if isinstance(ref, list): if isinstance(ref, list):
@ -142,7 +142,7 @@ def rm():
if DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id): if DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id):
break break
else: else:
return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR) return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=RetCode.OPERATING_ERROR)
ConversationService.delete_by_id(cid) ConversationService.delete_by_id(cid)
return get_json_result(data=True) return get_json_result(data=True)
except Exception as e: except Exception as e:
@ -155,7 +155,7 @@ def list_conversation():
dialog_id = request.args["dialog_id"] dialog_id = request.args["dialog_id"]
try: try:
if not DialogService.query(tenant_id=current_user.id, id=dialog_id): if not DialogService.query(tenant_id=current_user.id, id=dialog_id):
return get_json_result(data=False, message="Only owner of dialog authorized for this operation.", code=settings.RetCode.OPERATING_ERROR) return get_json_result(data=False, message="Only owner of dialog authorized for this operation.", code=RetCode.OPERATING_ERROR)
convs = ConversationService.query(dialog_id=dialog_id, order_by=ConversationService.model.create_time, reverse=True) convs = ConversationService.query(dialog_id=dialog_id, order_by=ConversationService.model.create_time, reverse=True)
convs = [d.to_dict() for d in convs] convs = [d.to_dict() for d in convs]

4
api/apps/dialog_app.py
View File

@ -22,9 +22,9 @@ from api.db import StatusEnum
from api.db.services.tenant_llm_service import TenantLLMService from api.db.services.tenant_llm_service import TenantLLMService
from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.user_service import TenantService, UserTenantService from api.db.services.user_service import TenantService, UserTenantService
from api import settings
from api.utils.api_utils import server_error_response, get_data_error_result, validate_request from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.contants import RetCode
from api.utils.api_utils import get_json_result from api.utils.api_utils import get_json_result
@ -219,7 +219,7 @@ def rm():
else: else:
return get_json_result( return get_json_result(
data=False, message='Only owner of dialog authorized for this operation.', data=False, message='Only owner of dialog authorized for this operation.',
code=settings.RetCode.OPERATING_ERROR) code=RetCode.OPERATING_ERROR)
dialog_list.append({"id": id,"status":StatusEnum.INVALID.value}) dialog_list.append({"id": id,"status":StatusEnum.INVALID.value})
DialogService.update_many_by_id(dialog_list) DialogService.update_many_by_id(dialog_list)
return get_json_result(data=True) return get_json_result(data=True)

75
api/apps/document_app.py
View File

@ -44,6 +44,7 @@ from api.utils.api_utils import (
) )
from api.utils.file_utils import filename_type, thumbnail from api.utils.file_utils import filename_type, thumbnail
from common.file_utils import get_project_base_directory from common.file_utils import get_project_base_directory
from common.contants import RetCode
from api.utils.web_utils import CONTENT_TYPE_MAP, html2pdf, is_valid_url from api.utils.web_utils import CONTENT_TYPE_MAP, html2pdf, is_valid_url
from deepdoc.parser.html_parser import RAGFlowHtmlParser from deepdoc.parser.html_parser import RAGFlowHtmlParser
from rag.nlp import search, rag_tokenizer from rag.nlp import search, rag_tokenizer
@ -56,29 +57,29 @@ from rag.utils.storage_factory import STORAGE_IMPL
def upload(): def upload():
kb_id = request.form.get("kb_id") kb_id = request.form.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
if "file" not in request.files: if "file" not in request.files:
return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist("file") file_objs = request.files.getlist("file")
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == "": if file_obj.filename == "":
return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR)
if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT: if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
e, kb = KnowledgebaseService.get_by_id(kb_id) e, kb = KnowledgebaseService.get_by_id(kb_id)
if not e: if not e:
raise LookupError("Can't find this knowledgebase!") raise LookupError("Can't find this knowledgebase!")
if not check_kb_team_permission(kb, current_user.id): if not check_kb_team_permission(kb, current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
err, files = FileService.upload_document(kb, file_objs, current_user.id) err, files = FileService.upload_document(kb, file_objs, current_user.id)
if err: if err:
return get_json_result(data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR) return get_json_result(data=files, message="\n".join(err), code=RetCode.SERVER_ERROR)
if not files: if not files:
return get_json_result(data=files, message="There seems to be an issue with your file format. Please verify it is correct and not corrupted.", code=settings.RetCode.DATA_ERROR) return get_json_result(data=files, message="There seems to be an issue with your file format. Please verify it is correct and not corrupted.", code=RetCode.DATA_ERROR)
files = [f[0] for f in files] # remove the blob files = [f[0] for f in files] # remove the blob
return get_json_result(data=files) return get_json_result(data=files)
@ -90,16 +91,16 @@ def upload():
def web_crawl(): def web_crawl():
kb_id = request.form.get("kb_id") kb_id = request.form.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
name = request.form.get("name") name = request.form.get("name")
url = request.form.get("url") url = request.form.get("url")
if not is_valid_url(url): if not is_valid_url(url):
return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
e, kb = KnowledgebaseService.get_by_id(kb_id) e, kb = KnowledgebaseService.get_by_id(kb_id)
if not e: if not e:
raise LookupError("Can't find this knowledgebase!") raise LookupError("Can't find this knowledgebase!")
if check_kb_team_permission(kb, current_user.id): if check_kb_team_permission(kb, current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
blob = html2pdf(url) blob = html2pdf(url)
if not blob: if not blob:
@ -156,12 +157,12 @@ def create():
req = request.json req = request.json
kb_id = req["kb_id"] kb_id = req["kb_id"]
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT: if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
if req["name"].strip() == "": if req["name"].strip() == "":
return get_json_result(data=False, message="File name can't be empty.", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="File name can't be empty.", code=RetCode.ARGUMENT_ERROR)
req["name"] = req["name"].strip() req["name"] = req["name"].strip()
try: try:
@ -211,13 +212,13 @@ def create():
def list_docs(): def list_docs():
kb_id = request.args.get("kb_id") kb_id = request.args.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
tenants = UserTenantService.query(user_id=current_user.id) tenants = UserTenantService.query(user_id=current_user.id)
for tenant in tenants: for tenant in tenants:
if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id): if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
break break
else: else:
return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=settings.RetCode.OPERATING_ERROR) return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
keywords = request.args.get("keywords", "") keywords = request.args.get("keywords", "")
page_number = int(request.args.get("page", 0)) page_number = int(request.args.get("page", 0))
@ -273,13 +274,13 @@ def get_filter():
kb_id = req.get("kb_id") kb_id = req.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
tenants = UserTenantService.query(user_id=current_user.id) tenants = UserTenantService.query(user_id=current_user.id)
for tenant in tenants: for tenant in tenants:
if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id): if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
break break
else: else:
return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=settings.RetCode.OPERATING_ERROR) return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
keywords = req.get("keywords", "") keywords = req.get("keywords", "")
@ -311,7 +312,7 @@ def docinfos():
doc_ids = req["doc_ids"] doc_ids = req["doc_ids"]
for doc_id in doc_ids: for doc_id in doc_ids:
if not DocumentService.accessible(doc_id, current_user.id): if not DocumentService.accessible(doc_id, current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
docs = DocumentService.get_by_ids(doc_ids) docs = DocumentService.get_by_ids(doc_ids)
return get_json_result(data=list(docs.dicts())) return get_json_result(data=list(docs.dicts()))
@ -321,7 +322,7 @@ def docinfos():
def thumbnails(): def thumbnails():
doc_ids = request.args.getlist("doc_ids") doc_ids = request.args.getlist("doc_ids")
if not doc_ids: if not doc_ids:
return get_json_result(data=False, message='Lack of "Document ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "Document ID"', code=RetCode.ARGUMENT_ERROR)
try: try:
docs = DocumentService.get_thumbnails(doc_ids) docs = DocumentService.get_thumbnails(doc_ids)
@ -344,7 +345,7 @@ def change_status():
status = str(req.get("status", "")) status = str(req.get("status", ""))
if status not in ["0", "1"]: if status not in ["0", "1"]:
return get_json_result(data=False, message='"Status" must be either 0 or 1!', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='"Status" must be either 0 or 1!', code=RetCode.ARGUMENT_ERROR)
result = {} result = {}
for doc_id in doc_ids: for doc_id in doc_ids:
@ -386,12 +387,12 @@ def rm():
for doc_id in doc_ids: for doc_id in doc_ids:
if not DocumentService.accessible4deletion(doc_id, current_user.id): if not DocumentService.accessible4deletion(doc_id, current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
errors = FileService.delete_docs(doc_ids, current_user.id) errors = FileService.delete_docs(doc_ids, current_user.id)
if errors: if errors:
return get_json_result(data=False, message=errors, code=settings.RetCode.SERVER_ERROR) return get_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
return get_json_result(data=True) return get_json_result(data=True)
@ -403,7 +404,7 @@ def run():
req = request.json req = request.json
for doc_id in req["doc_ids"]: for doc_id in req["doc_ids"]:
if not DocumentService.accessible(doc_id, current_user.id): if not DocumentService.accessible(doc_id, current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
try: try:
kb_table_num_map = {} kb_table_num_map = {}
for id in req["doc_ids"]: for id in req["doc_ids"]:
@ -449,15 +450,15 @@ def run():
def rename(): def rename():
req = request.json req = request.json
if not DocumentService.accessible(req["doc_id"], current_user.id): if not DocumentService.accessible(req["doc_id"], current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
try: try:
e, doc = DocumentService.get_by_id(req["doc_id"]) e, doc = DocumentService.get_by_id(req["doc_id"])
if not e: if not e:
return get_data_error_result(message="Document not found!") return get_data_error_result(message="Document not found!")
if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix: if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
return get_json_result(data=False, message="The extension of file can't be changed", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="The extension of file can't be changed", code=RetCode.ARGUMENT_ERROR)
if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT: if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id): for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
if d.name == req["name"]: if d.name == req["name"]:
@ -522,7 +523,7 @@ def change_parser():
req = request.json req = request.json
if not DocumentService.accessible(req["doc_id"], current_user.id): if not DocumentService.accessible(req["doc_id"], current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
e, doc = DocumentService.get_by_id(req["doc_id"]) e, doc = DocumentService.get_by_id(req["doc_id"])
if not e: if not e:
@ -588,12 +589,12 @@ def get_image(image_id):
@validate_request("conversation_id") @validate_request("conversation_id")
def upload_and_parse(): def upload_and_parse():
if "file" not in request.files: if "file" not in request.files:
return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist("file") file_objs = request.files.getlist("file")
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == "": if file_obj.filename == "":
return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR)
doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, current_user.id) doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, current_user.id)
@ -606,7 +607,7 @@ def parse():
url = request.json.get("url") if request.json else "" url = request.json.get("url") if request.json else ""
if url: if url:
if not is_valid_url(url): if not is_valid_url(url):
return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
download_path = os.path.join(get_project_base_directory(), "logs/downloads") download_path = os.path.join(get_project_base_directory(), "logs/downloads")
os.makedirs(download_path, exist_ok=True) os.makedirs(download_path, exist_ok=True)
from seleniumwire.webdriver import Chrome, ChromeOptions from seleniumwire.webdriver import Chrome, ChromeOptions
@ -639,13 +640,13 @@ def parse():
r = re.search(r"filename=\"([^\"]+)\"", str(res_headers)) r = re.search(r"filename=\"([^\"]+)\"", str(res_headers))
if not r or not r.group(1): if not r or not r.group(1):
return get_json_result(data=False, message="Can't not identify downloaded file", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="Can't not identify downloaded file", code=RetCode.ARGUMENT_ERROR)
f = File(r.group(1), os.path.join(download_path, r.group(1))) f = File(r.group(1), os.path.join(download_path, r.group(1)))
txt = FileService.parse_docs([f], current_user.id) txt = FileService.parse_docs([f], current_user.id)
return get_json_result(data=txt) return get_json_result(data=txt)
if "file" not in request.files: if "file" not in request.files:
return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist("file") file_objs = request.files.getlist("file")
txt = FileService.parse_docs(file_objs, current_user.id) txt = FileService.parse_docs(file_objs, current_user.id)
@ -659,18 +660,18 @@ def parse():
def set_meta(): def set_meta():
req = request.json req = request.json
if not DocumentService.accessible(req["doc_id"], current_user.id): if not DocumentService.accessible(req["doc_id"], current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
try: try:
meta = json.loads(req["meta"]) meta = json.loads(req["meta"])
if not isinstance(meta, dict): if not isinstance(meta, dict):
return get_json_result(data=False, message="Only dictionary type supported.", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="Only dictionary type supported.", code=RetCode.ARGUMENT_ERROR)
for k, v in meta.items(): for k, v in meta.items():
if not isinstance(v, str) and not isinstance(v, int) and not isinstance(v, float): if not isinstance(v, str) and not isinstance(v, int) and not isinstance(v, float):
return get_json_result(data=False, message=f"The type is not supported: {v}", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message=f"The type is not supported: {v}", code=RetCode.ARGUMENT_ERROR)
except Exception as e: except Exception as e:
return get_json_result(data=False, message=f"Json syntax error: {e}", code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message=f"Json syntax error: {e}", code=RetCode.ARGUMENT_ERROR)
if not isinstance(meta, dict): if not isinstance(meta, dict):
return get_json_result(data=False, message='Meta data should be in Json map format, like {"key": "value"}', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Meta data should be in Json map format, like {"key": "value"}', code=RetCode.ARGUMENT_ERROR)
try: try:
e, doc = DocumentService.get_by_id(req["doc_id"]) e, doc = DocumentService.get_by_id(req["doc_id"])

4
api/apps/file2document_app.py
View File

@ -24,9 +24,9 @@ from flask_login import login_required, current_user
from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.knowledgebase_service import KnowledgebaseService
from api.utils.api_utils import server_error_response, get_data_error_result, validate_request from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.contants import RetCode
from api.db import FileType from api.db import FileType
from api.db.services.document_service import DocumentService from api.db.services.document_service import DocumentService
from api import settings
from api.utils.api_utils import get_json_result from api.utils.api_utils import get_json_result
@ -108,7 +108,7 @@ def rm():
file_ids = req["file_ids"] file_ids = req["file_ids"]
if not file_ids: if not file_ids:
return get_json_result( return get_json_result(
data=False, message='Lack of "Files ID"', code=settings.RetCode.ARGUMENT_ERROR) data=False, message='Lack of "Files ID"', code=RetCode.ARGUMENT_ERROR)
try: try:
for file_id in file_ids: for file_id in file_ids:
informs = File2DocumentService.get_by_file_id(file_id) informs = File2DocumentService.get_by_file_id(file_id)

18
api/apps/file_app.py
View File

@ -27,10 +27,10 @@ from api.db.services.document_service import DocumentService
from api.db.services.file2document_service import File2DocumentService from api.db.services.file2document_service import File2DocumentService
from api.utils.api_utils import server_error_response, get_data_error_result, validate_request from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.contants import RetCode
from api.db import FileType, FileSource from api.db import FileType, FileSource
from api.db.services import duplicate_name from api.db.services import duplicate_name
from api.db.services.file_service import FileService from api.db.services.file_service import FileService
from api import settings
from api.utils.api_utils import get_json_result from api.utils.api_utils import get_json_result
from api.utils.file_utils import filename_type from api.utils.file_utils import filename_type
from api.utils.web_utils import CONTENT_TYPE_MAP from api.utils.web_utils import CONTENT_TYPE_MAP
@ -49,13 +49,13 @@ def upload():
if 'file' not in request.files: if 'file' not in request.files:
return get_json_result( return get_json_result(
data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR) data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist('file') file_objs = request.files.getlist('file')
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == '': if file_obj.filename == '':
return get_json_result( return get_json_result(
data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR) data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR)
file_res = [] file_res = []
try: try:
e, pf_folder = FileService.get_by_id(pf_id) e, pf_folder = FileService.get_by_id(pf_id)
@ -134,7 +134,7 @@ def create():
try: try:
if not FileService.is_parent_folder_exist(pf_id): if not FileService.is_parent_folder_exist(pf_id):
return get_json_result( return get_json_result(
data=False, message="Parent Folder Doesn't Exist!", code=settings.RetCode.OPERATING_ERROR) data=False, message="Parent Folder Doesn't Exist!", code=RetCode.OPERATING_ERROR)
if FileService.query(name=req["name"], parent_id=pf_id): if FileService.query(name=req["name"], parent_id=pf_id):
return get_data_error_result( return get_data_error_result(
message="Duplicated folder name in the same folder.") message="Duplicated folder name in the same folder.")
@ -279,7 +279,7 @@ def rm():
if not file.tenant_id: if not file.tenant_id:
return get_data_error_result(message="Tenant not found!") return get_data_error_result(message="Tenant not found!")
if not check_file_team_permission(file, current_user.id): if not check_file_team_permission(file, current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
if file.source_type == FileSource.KNOWLEDGEBASE: if file.source_type == FileSource.KNOWLEDGEBASE:
continue continue
@ -306,14 +306,14 @@ def rename():
if not e: if not e:
return get_data_error_result(message="File not found!") return get_data_error_result(message="File not found!")
if not check_file_team_permission(file, current_user.id): if not check_file_team_permission(file, current_user.id):
return get_json_result(data=False, message='No authorization.', code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message='No authorization.', code=RetCode.AUTHENTICATION_ERROR)
if file.type != FileType.FOLDER.value \ if file.type != FileType.FOLDER.value \
and pathlib.Path(req["name"].lower()).suffix != pathlib.Path( and pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
file.name.lower()).suffix: file.name.lower()).suffix:
return get_json_result( return get_json_result(
data=False, data=False,
message="The extension of file can't be changed", message="The extension of file can't be changed",
code=settings.RetCode.ARGUMENT_ERROR) code=RetCode.ARGUMENT_ERROR)
for file in FileService.query(name=req["name"], pf_id=file.parent_id): for file in FileService.query(name=req["name"], pf_id=file.parent_id):
if file.name == req["name"]: if file.name == req["name"]:
return get_data_error_result( return get_data_error_result(
@ -344,7 +344,7 @@ def get(file_id):
if not e: if not e:
return get_data_error_result(message="Document not found!") return get_data_error_result(message="Document not found!")
if not check_file_team_permission(file, current_user.id): if not check_file_team_permission(file, current_user.id):
return get_json_result(data=False, message='No authorization.', code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message='No authorization.', code=RetCode.AUTHENTICATION_ERROR)
blob = STORAGE_IMPL.get(file.parent_id, file.location) blob = STORAGE_IMPL.get(file.parent_id, file.location)
if not blob: if not blob:
@ -394,7 +394,7 @@ def move():
return get_json_result( return get_json_result(
data=False, data=False,
message="No authorization.", message="No authorization.",
code=settings.RetCode.AUTHENTICATION_ERROR, code=RetCode.AUTHENTICATION_ERROR,
) )
def _move_entry_recursive(source_file_entry, dest_folder): def _move_entry_recursive(source_file_entry, dest_folder):

37
api/apps/kb_app.py
View File

@ -41,6 +41,7 @@ from rag.settings import PAGERANK_FLD
from rag.utils.redis_conn import REDIS_CONN from rag.utils.redis_conn import REDIS_CONN
from rag.utils.storage_factory import STORAGE_IMPL from rag.utils.storage_factory import STORAGE_IMPL
from rag.utils.doc_store_conn import OrderByExpr from rag.utils.doc_store_conn import OrderByExpr
from common.contants import RetCode
@manager.route('/create', methods=['post']) # noqa: F821 @manager.route('/create', methods=['post']) # noqa: F821
@ -82,14 +83,14 @@ def update():
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
try: try:
if not KnowledgebaseService.query( if not KnowledgebaseService.query(
created_by=current_user.id, id=req["kb_id"]): created_by=current_user.id, id=req["kb_id"]):
return get_json_result( return get_json_result(
data=False, message='Only owner of knowledgebase authorized for this operation.', data=False, message='Only owner of knowledgebase authorized for this operation.',
code=settings.RetCode.OPERATING_ERROR) code=RetCode.OPERATING_ERROR)
e, kb = KnowledgebaseService.get_by_id(req["kb_id"]) e, kb = KnowledgebaseService.get_by_id(req["kb_id"])
if not e: if not e:
@ -140,7 +141,7 @@ def detail():
else: else:
return get_json_result( return get_json_result(
data=False, message='Only owner of knowledgebase authorized for this operation.', data=False, message='Only owner of knowledgebase authorized for this operation.',
code=settings.RetCode.OPERATING_ERROR) code=RetCode.OPERATING_ERROR)
kb = KnowledgebaseService.get_detail(kb_id) kb = KnowledgebaseService.get_detail(kb_id)
if not kb: if not kb:
return get_data_error_result( return get_data_error_result(
@ -198,7 +199,7 @@ def rm():
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
try: try:
kbs = KnowledgebaseService.query( kbs = KnowledgebaseService.query(
@ -206,7 +207,7 @@ def rm():
if not kbs: if not kbs:
return get_json_result( return get_json_result(
data=False, message='Only owner of knowledgebase authorized for this operation.', data=False, message='Only owner of knowledgebase authorized for this operation.',
code=settings.RetCode.OPERATING_ERROR) code=RetCode.OPERATING_ERROR)
for doc in DocumentService.query(kb_id=req["kb_id"]): for doc in DocumentService.query(kb_id=req["kb_id"]):
if not DocumentService.remove_document(doc, kbs[0].tenant_id): if not DocumentService.remove_document(doc, kbs[0].tenant_id):
@ -238,7 +239,7 @@ def list_tags(kb_id):
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
tenants = UserTenantService.get_tenants_by_user_id(current_user.id) tenants = UserTenantService.get_tenants_by_user_id(current_user.id)
@ -257,7 +258,7 @@ def list_tags_from_kbs():
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
tenants = UserTenantService.get_tenants_by_user_id(current_user.id) tenants = UserTenantService.get_tenants_by_user_id(current_user.id)
@ -275,7 +276,7 @@ def rm_tags(kb_id):
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
e, kb = KnowledgebaseService.get_by_id(kb_id) e, kb = KnowledgebaseService.get_by_id(kb_id)
@ -295,7 +296,7 @@ def rename_tags(kb_id):
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
e, kb = KnowledgebaseService.get_by_id(kb_id) e, kb = KnowledgebaseService.get_by_id(kb_id)
@ -313,7 +314,7 @@ def knowledge_graph(kb_id):
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
_, kb = KnowledgebaseService.get_by_id(kb_id) _, kb = KnowledgebaseService.get_by_id(kb_id)
req = { req = {
@ -353,7 +354,7 @@ def delete_knowledge_graph(kb_id):
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
_, kb = KnowledgebaseService.get_by_id(kb_id) _, kb = KnowledgebaseService.get_by_id(kb_id)
settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), kb_id) settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), kb_id)
@ -370,7 +371,7 @@ def get_meta():
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
return get_json_result(data=DocumentService.get_meta_by_kbs(kb_ids)) return get_json_result(data=DocumentService.get_meta_by_kbs(kb_ids))
@ -383,7 +384,7 @@ def get_basic_info():
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
basic_info = DocumentService.knowledgebase_basic_info(kb_id) basic_info = DocumentService.knowledgebase_basic_info(kb_id)
@ -396,7 +397,7 @@ def get_basic_info():
def list_pipeline_logs(): def list_pipeline_logs():
kb_id = request.args.get("kb_id") kb_id = request.args.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
keywords = request.args.get("keywords", "") keywords = request.args.get("keywords", "")
@ -440,7 +441,7 @@ def list_pipeline_logs():
def list_pipeline_dataset_logs(): def list_pipeline_dataset_logs():
kb_id = request.args.get("kb_id") kb_id = request.args.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
page_number = int(request.args.get("page", 0)) page_number = int(request.args.get("page", 0))
items_per_page = int(request.args.get("page_size", 0)) items_per_page = int(request.args.get("page_size", 0))
@ -474,7 +475,7 @@ def list_pipeline_dataset_logs():
def delete_pipeline_logs(): def delete_pipeline_logs():
kb_id = request.args.get("kb_id") kb_id = request.args.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
req = request.get_json() req = request.get_json()
log_ids = req.get("log_ids", []) log_ids = req.get("log_ids", [])
@ -489,7 +490,7 @@ def delete_pipeline_logs():
def pipeline_log_detail(): def pipeline_log_detail():
log_id = request.args.get("log_id") log_id = request.args.get("log_id")
if not log_id: if not log_id:
return get_json_result(data=False, message='Lack of "Pipeline log ID"', code=settings.RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "Pipeline log ID"', code=RetCode.ARGUMENT_ERROR)
ok, log = PipelineOperationLogService.get_by_id(log_id) ok, log = PipelineOperationLogService.get_by_id(log_id)
if not ok: if not ok:
@ -882,4 +883,4 @@ def check_embedding():
} }
if summary["avg_cos_sim"] > 0.99: if summary["avg_cos_sim"] > 0.99:
return get_json_result(data={"summary": summary, "results": results}) return get_json_result(data={"summary": summary, "results": results})
return get_json_result(code=settings.RetCode.NOT_EFFECTIVE, message="failed", data={"summary": summary, "results": results}) return get_json_result(code=RetCode.NOT_EFFECTIVE, message="failed", data={"summary": summary, "results": results})

2
api/apps/mcp_server_app.py
View File

@ -20,7 +20,7 @@ from api.db import VALID_MCP_SERVER_TYPES
from api.db.db_models import MCPServer from api.db.db_models import MCPServer
from api.db.services.mcp_server_service import MCPServerService from api.db.services.mcp_server_service import MCPServerService
from api.db.services.user_service import TenantService from api.db.services.user_service import TenantService
from api.settings import RetCode from common.contants import RetCode
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request, \ from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request, \

2
api/apps/sdk/agent.py
View File

@ -19,7 +19,7 @@ import time
from typing import Any, cast from typing import Any, cast
from api.db.services.canvas_service import UserCanvasService from api.db.services.canvas_service import UserCanvasService
from api.db.services.user_canvas_version import UserCanvasVersionService from api.db.services.user_canvas_version import UserCanvasVersionService
from api.settings import RetCode from common.contants import RetCode
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from api.utils.api_utils import get_data_error_result, get_error_data_result, get_json_result, token_required from api.utils.api_utils import get_data_error_result, get_error_data_result, get_json_result, token_required
from api.utils.api_utils import get_result from api.utils.api_utils import get_result

6
api/apps/sdk/chat.py
View File

@ -17,13 +17,13 @@ import logging
from flask import request from flask import request
from api import settings
from api.db import StatusEnum from api.db import StatusEnum
from api.db.services.dialog_service import DialogService from api.db.services.dialog_service import DialogService
from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.tenant_llm_service import TenantLLMService from api.db.services.tenant_llm_service import TenantLLMService
from api.db.services.user_service import TenantService from api.db.services.user_service import TenantService
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.contants import RetCode
from api.utils.api_utils import check_duplicate_ids, get_error_data_result, get_result, token_required from api.utils.api_utils import check_duplicate_ids, get_error_data_result, get_result, token_required
@ -45,7 +45,7 @@ def create(tenant_id):
embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs] # remove vendor suffix for comparison embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs] # remove vendor suffix for comparison
embd_count = list(set(embd_ids)) embd_count = list(set(embd_ids))
if len(embd_count) > 1: if len(embd_count) > 1:
return get_result(message='Datasets use different embedding models."', code=settings.RetCode.AUTHENTICATION_ERROR) return get_result(message='Datasets use different embedding models."', code=RetCode.AUTHENTICATION_ERROR)
req["kb_ids"] = ids req["kb_ids"] = ids
# llm # llm
llm = req.get("llm") llm = req.get("llm")
@ -167,7 +167,7 @@ def update(tenant_id, chat_id):
embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs] # remove vendor suffix for comparison embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs] # remove vendor suffix for comparison
embd_count = list(set(embd_ids)) embd_count = list(set(embd_ids))
if len(embd_count) > 1: if len(embd_count) > 1:
return get_result(message='Datasets use different embedding models."', code=settings.RetCode.AUTHENTICATION_ERROR) return get_result(message='Datasets use different embedding models."', code=RetCode.AUTHENTICATION_ERROR)
req["kb_ids"] = ids req["kb_ids"] = ids
else: else:
req["kb_ids"] = [] req["kb_ids"] = []

5
api/apps/sdk/dataset.py
View File

@ -28,6 +28,7 @@ from api.db.services.file2document_service import File2DocumentService
from api.db.services.file_service import FileService from api.db.services.file_service import FileService
from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.user_service import TenantService from api.db.services.user_service import TenantService
from common.contants import RetCode
from api.utils.api_utils import ( from api.utils.api_utils import (
deep_merge, deep_merge,
get_error_argument_result, get_error_argument_result,
@ -484,7 +485,7 @@ def knowledge_graph(tenant_id, dataset_id):
return get_result( return get_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
_, kb = KnowledgebaseService.get_by_id(dataset_id) _, kb = KnowledgebaseService.get_by_id(dataset_id)
req = { req = {
@ -525,7 +526,7 @@ def delete_knowledge_graph(tenant_id, dataset_id):
return get_result( return get_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR code=RetCode.AUTHENTICATION_ERROR
) )
_, kb = KnowledgebaseService.get_by_id(dataset_id) _, kb = KnowledgebaseService.get_by_id(dataset_id)
settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]},

8
api/apps/sdk/dify_retrieval.py
View File

@ -25,7 +25,7 @@ from api import settings
from api.utils.api_utils import validate_request, build_error_result, apikey_required from api.utils.api_utils import validate_request, build_error_result, apikey_required
from rag.app.tag import label_question from rag.app.tag import label_question
from api.db.services.dialog_service import meta_filter, convert_conditions from api.db.services.dialog_service import meta_filter, convert_conditions
from common.contants import RetCode
@manager.route('/dify/retrieval', methods=['POST']) # noqa: F821 @manager.route('/dify/retrieval', methods=['POST']) # noqa: F821
@apikey_required @apikey_required
@ -129,7 +129,7 @@ def retrieval(tenant_id):
e, kb = KnowledgebaseService.get_by_id(kb_id) e, kb = KnowledgebaseService.get_by_id(kb_id)
if not e: if not e:
return build_error_result(message="Knowledgebase not found!", code=settings.RetCode.NOT_FOUND) return build_error_result(message="Knowledgebase not found!", code=RetCode.NOT_FOUND)
embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id) embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
print(metadata_condition) print(metadata_condition)
@ -179,7 +179,7 @@ def retrieval(tenant_id):
if str(e).find("not_found") > 0: if str(e).find("not_found") > 0:
return build_error_result( return build_error_result(
message='No chunk found! Check the chunk status please!', message='No chunk found! Check the chunk status please!',
code=settings.RetCode.NOT_FOUND code=RetCode.NOT_FOUND
) )
logging.exception(e) logging.exception(e)
return build_error_result(message=str(e), code=settings.RetCode.SERVER_ERROR) return build_error_result(message=str(e), code=RetCode.SERVER_ERROR)

29
api/apps/sdk/doc.py
View File

@ -43,6 +43,7 @@ from rag.nlp import rag_tokenizer, search
from rag.prompts.generator import cross_languages, keyword_extraction from rag.prompts.generator import cross_languages, keyword_extraction
from rag.utils.storage_factory import STORAGE_IMPL from rag.utils.storage_factory import STORAGE_IMPL
from common.string_utils import remove_redundant_spaces from common.string_utils import remove_redundant_spaces
from common.contants import RetCode
MAXIMUM_OF_UPLOADING_FILES = 256 MAXIMUM_OF_UPLOADING_FILES = 256
@ -127,13 +128,13 @@ def upload(dataset_id, tenant_id):
description: Processing status. description: Processing status.
""" """
if "file" not in request.files: if "file" not in request.files:
return get_error_data_result(message="No file part!", code=settings.RetCode.ARGUMENT_ERROR) return get_error_data_result(message="No file part!", code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist("file") file_objs = request.files.getlist("file")
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == "": if file_obj.filename == "":
return get_result(message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR) return get_result(message="No file selected!", code=RetCode.ARGUMENT_ERROR)
if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT: if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
return get_result(message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR) return get_result(message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
""" """
# total size # total size
total_size = 0 total_size = 0
@ -145,7 +146,7 @@ def upload(dataset_id, tenant_id):
if total_size > MAX_TOTAL_FILE_SIZE: if total_size > MAX_TOTAL_FILE_SIZE:
return get_result( return get_result(
message=f"Total file size exceeds 10MB limit! ({total_size / (1024 * 1024):.2f} MB)", message=f"Total file size exceeds 10MB limit! ({total_size / (1024 * 1024):.2f} MB)",
code=settings.RetCode.ARGUMENT_ERROR, code=RetCode.ARGUMENT_ERROR,
) )
""" """
e, kb = KnowledgebaseService.get_by_id(dataset_id) e, kb = KnowledgebaseService.get_by_id(dataset_id)
@ -153,7 +154,7 @@ def upload(dataset_id, tenant_id):
raise LookupError(f"Can't find the dataset with ID {dataset_id}!") raise LookupError(f"Can't find the dataset with ID {dataset_id}!")
err, files = FileService.upload_document(kb, file_objs, tenant_id) err, files = FileService.upload_document(kb, file_objs, tenant_id)
if err: if err:
return get_result(message="\n".join(err), code=settings.RetCode.SERVER_ERROR) return get_result(message="\n".join(err), code=RetCode.SERVER_ERROR)
# rename key's name # rename key's name
renamed_doc_list = [] renamed_doc_list = []
for file in files: for file in files:
@ -253,12 +254,12 @@ def update_doc(tenant_id, dataset_id, document_id):
if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT: if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
return get_result( return get_result(
message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.",
code=settings.RetCode.ARGUMENT_ERROR, code=RetCode.ARGUMENT_ERROR,
) )
if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix: if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
return get_result( return get_result(
message="The extension of file can't be changed", message="The extension of file can't be changed",
code=settings.RetCode.ARGUMENT_ERROR, code=RetCode.ARGUMENT_ERROR,
) )
for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id): for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
if d.name == req["name"]: if d.name == req["name"]:
@ -402,7 +403,7 @@ def download(tenant_id, dataset_id, document_id):
doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id) # minio address doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id) # minio address
file_stream = STORAGE_IMPL.get(doc_id, doc_location) file_stream = STORAGE_IMPL.get(doc_id, doc_location)
if not file_stream: if not file_stream:
return construct_json_result(message="This file is empty.", code=settings.RetCode.DATA_ERROR) return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR)
file = BytesIO(file_stream) file = BytesIO(file_stream)
# Use send_file with a proper filename and MIME type # Use send_file with a proper filename and MIME type
return send_file( return send_file(
@ -676,10 +677,10 @@ def delete(tenant_id, dataset_id):
errors += str(e) errors += str(e)
if not_found: if not_found:
return get_result(message=f"Documents not found: {not_found}", code=settings.RetCode.DATA_ERROR) return get_result(message=f"Documents not found: {not_found}", code=RetCode.DATA_ERROR)
if errors: if errors:
return get_result(message=errors, code=settings.RetCode.SERVER_ERROR) return get_result(message=errors, code=RetCode.SERVER_ERROR)
if duplicate_messages: if duplicate_messages:
if success_count > 0: if success_count > 0:
@ -763,7 +764,7 @@ def parse(tenant_id, dataset_id):
queue_tasks(doc, bucket, name, 0) queue_tasks(doc, bucket, name, 0)
success_count += 1 success_count += 1
if not_found: if not_found:
return get_result(message=f"Documents not found: {not_found}", code=settings.RetCode.DATA_ERROR) return get_result(message=f"Documents not found: {not_found}", code=RetCode.DATA_ERROR)
if duplicate_messages: if duplicate_messages:
if success_count > 0: if success_count > 0:
return get_result( return get_result(
@ -969,7 +970,7 @@ def list_chunks(tenant_id, dataset_id, document_id):
if req.get("id"): if req.get("id"):
chunk = settings.docStoreConn.get(req.get("id"), search.index_name(tenant_id), [dataset_id]) chunk = settings.docStoreConn.get(req.get("id"), search.index_name(tenant_id), [dataset_id])
if not chunk: if not chunk:
return get_result(message=f"Chunk not found: {dataset_id}/{req.get('id')}", code=settings.RetCode.NOT_FOUND) return get_result(message=f"Chunk not found: {dataset_id}/{req.get('id')}", code=RetCode.NOT_FOUND)
k = [] k = []
for n in chunk.keys(): for n in chunk.keys():
if re.search(r"(_vec$|_sm_|_tks|_ltks)", n): if re.search(r"(_vec$|_sm_|_tks|_ltks)", n):
@ -1418,7 +1419,7 @@ def retrieval_test(tenant_id):
if len(embd_nms) != 1: if len(embd_nms) != 1:
return get_result( return get_result(
message='Datasets use different embedding models."', message='Datasets use different embedding models."',
code=settings.RetCode.DATA_ERROR, code=RetCode.DATA_ERROR,
) )
if "question" not in req: if "question" not in req:
return get_error_data_result("`question` is required.") return get_error_data_result("`question` is required.")
@ -1509,6 +1510,6 @@ def retrieval_test(tenant_id):
if str(e).find("not_found") > 0: if str(e).find("not_found") > 0:
return get_result( return get_result(
message="No chunk found! Check the chunk status please!", message="No chunk found! Check the chunk status please!",
code=settings.RetCode.DATA_ERROR, code=RetCode.DATA_ERROR,
) )
return server_error_response(e) return server_error_response(e)

10
api/apps/sdk/session.py
View File

@ -41,7 +41,7 @@ from api.utils.api_utils import check_duplicate_ids, get_data_openai, get_error_
from rag.app.tag import label_question from rag.app.tag import label_question
from rag.prompts.template import load_prompt from rag.prompts.template import load_prompt
from rag.prompts.generator import cross_languages, gen_meta_filter, keyword_extraction, chunks_format from rag.prompts.generator import cross_languages, gen_meta_filter, keyword_extraction, chunks_format
from common.contants import RetCode
@manager.route("/chats/<chat_id>/sessions", methods=["POST"]) # noqa: F821 @manager.route("/chats/<chat_id>/sessions", methods=["POST"]) # noqa: F821
@token_required @token_required
@ -959,7 +959,7 @@ def retrieval_test_embedded():
kb_ids = [kb_ids] kb_ids = [kb_ids]
if not kb_ids: if not kb_ids:
return get_json_result(data=False, message='Please specify dataset firstly.', return get_json_result(data=False, message='Please specify dataset firstly.',
code=settings.RetCode.DATA_ERROR) code=RetCode.DATA_ERROR)
doc_ids = req.get("doc_ids", []) doc_ids = req.get("doc_ids", [])
similarity_threshold = float(req.get("similarity_threshold", 0.0)) similarity_threshold = float(req.get("similarity_threshold", 0.0))
vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3)) vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
@ -996,7 +996,7 @@ def retrieval_test_embedded():
break break
else: else:
return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.",
code=settings.RetCode.OPERATING_ERROR) code=RetCode.OPERATING_ERROR)
e, kb = KnowledgebaseService.get_by_id(kb_ids[0]) e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
if not e: if not e:
@ -1034,7 +1034,7 @@ def retrieval_test_embedded():
except Exception as e: except Exception as e:
if str(e).find("not_found") > 0: if str(e).find("not_found") > 0:
return get_json_result(data=False, message="No chunk found! Check the chunk status please!", return get_json_result(data=False, message="No chunk found! Check the chunk status please!",
code=settings.RetCode.DATA_ERROR) code=RetCode.DATA_ERROR)
return server_error_response(e) return server_error_response(e)
@ -1104,7 +1104,7 @@ def detail_share_embedded():
break break
else: else:
return get_json_result(data=False, message="Has no permission for this operation.", return get_json_result(data=False, message="Has no permission for this operation.",
code=settings.RetCode.OPERATING_ERROR) code=RetCode.OPERATING_ERROR)
search = SearchService.get_detail(search_id) search = SearchService.get_detail(search_id)
if not search: if not search:

10
api/apps/search_app.py
View File

@ -17,7 +17,6 @@
from flask import request from flask import request
from flask_login import current_user, login_required from flask_login import current_user, login_required
from api import settings
from api.constants import DATASET_NAME_LIMIT from api.constants import DATASET_NAME_LIMIT
from api.db import StatusEnum from api.db import StatusEnum
from api.db.db_models import DB from api.db.db_models import DB
@ -25,6 +24,7 @@ from api.db.services import duplicate_name
from api.db.services.search_service import SearchService from api.db.services.search_service import SearchService
from api.db.services.user_service import TenantService, UserTenantService from api.db.services.user_service import TenantService, UserTenantService
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.contants import RetCode
from api.utils.api_utils import get_data_error_result, get_json_result, not_allowed_parameters, server_error_response, validate_request from api.utils.api_utils import get_data_error_result, get_json_result, not_allowed_parameters, server_error_response, validate_request
@ -82,12 +82,12 @@ def update():
search_id = req["search_id"] search_id = req["search_id"]
if not SearchService.accessible4deletion(search_id, current_user.id): if not SearchService.accessible4deletion(search_id, current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
try: try:
search_app = SearchService.query(tenant_id=tenant_id, id=search_id)[0] search_app = SearchService.query(tenant_id=tenant_id, id=search_id)[0]
if not search_app: if not search_app:
return get_json_result(data=False, message=f"Cannot find search {search_id}", code=settings.RetCode.DATA_ERROR) return get_json_result(data=False, message=f"Cannot find search {search_id}", code=RetCode.DATA_ERROR)
if req["name"].lower() != search_app.name.lower() and len(SearchService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) >= 1: if req["name"].lower() != search_app.name.lower() and len(SearchService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) >= 1:
return get_data_error_result(message="Duplicated search name.") return get_data_error_result(message="Duplicated search name.")
@ -129,7 +129,7 @@ def detail():
if SearchService.query(tenant_id=tenant.tenant_id, id=search_id): if SearchService.query(tenant_id=tenant.tenant_id, id=search_id):
break break
else: else:
return get_json_result(data=False, message="Has no permission for this operation.", code=settings.RetCode.OPERATING_ERROR) return get_json_result(data=False, message="Has no permission for this operation.", code=RetCode.OPERATING_ERROR)
search = SearchService.get_detail(search_id) search = SearchService.get_detail(search_id)
if not search: if not search:
@ -178,7 +178,7 @@ def rm():
req = request.get_json() req = request.get_json()
search_id = req["search_id"] search_id = req["search_id"]
if not SearchService.accessible4deletion(search_id, current_user.id): if not SearchService.accessible4deletion(search_id, current_user.id):
return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
try: try:
if not SearchService.delete_by_id(search_id): if not SearchService.delete_by_id(search_id):

7
api/apps/tenant_app.py
View File

@ -23,6 +23,7 @@ from api.db import UserTenantRole, StatusEnum
from api.db.db_models import UserTenant from api.db.db_models import UserTenant
from api.db.services.user_service import UserTenantService, UserService from api.db.services.user_service import UserTenantService, UserService
from common.contants import RetCode
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.time_utils import delta_seconds from common.time_utils import delta_seconds
from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result
@ -36,7 +37,7 @@ def user_list(tenant_id):
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR) code=RetCode.AUTHENTICATION_ERROR)
try: try:
users = UserTenantService.get_by_tenant_id(tenant_id) users = UserTenantService.get_by_tenant_id(tenant_id)
@ -55,7 +56,7 @@ def create(tenant_id):
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR) code=RetCode.AUTHENTICATION_ERROR)
req = request.json req = request.json
invite_user_email = req["email"] invite_user_email = req["email"]
@ -109,7 +110,7 @@ def rm(tenant_id, user_id):
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=settings.RetCode.AUTHENTICATION_ERROR) code=RetCode.AUTHENTICATION_ERROR)
try: try:
UserTenantService.filter_delete([UserTenant.tenant_id == tenant_id, UserTenant.user_id == user_id]) UserTenantService.filter_delete([UserTenant.tenant_id == tenant_id, UserTenant.user_id == user_id])

61
api/apps/user_app.py
View File

@ -36,8 +36,9 @@ from api.db.services.tenant_llm_service import TenantLLMService
from api.db.services.user_service import TenantService, UserService, UserTenantService from api.db.services.user_service import TenantService, UserService, UserTenantService
from common.time_utils import current_timestamp, datetime_format, get_format_time from common.time_utils import current_timestamp, datetime_format, get_format_time
from common.misc_utils import download_img, get_uuid from common.misc_utils import download_img, get_uuid
from common.contants import RetCode
from common.connection_utils import construct_response
from api.utils.api_utils import ( from api.utils.api_utils import (
construct_response,
get_data_error_result, get_data_error_result,
get_json_result, get_json_result,
server_error_response, server_error_response,
@ -91,14 +92,14 @@ def login():
type: object type: object
""" """
if not request.json: if not request.json:
return get_json_result(data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="Unauthorized!") return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="Unauthorized!")
email = request.json.get("email", "") email = request.json.get("email", "")
users = UserService.query(email=email) users = UserService.query(email=email)
if not users: if not users:
return get_json_result( return get_json_result(
data=False, data=False,
code=settings.RetCode.AUTHENTICATION_ERROR, code=RetCode.AUTHENTICATION_ERROR,
message=f"Email: {email} is not registered!", message=f"Email: {email} is not registered!",
) )
@ -106,14 +107,14 @@ def login():
try: try:
password = decrypt(password) password = decrypt(password)
except BaseException: except BaseException:
return get_json_result(data=False, code=settings.RetCode.SERVER_ERROR, message="Fail to crypt password") return get_json_result(data=False, code=RetCode.SERVER_ERROR, message="Fail to crypt password")
user = UserService.query_user(email, password) user = UserService.query_user(email, password)
if user and hasattr(user, 'is_active') and user.is_active == "0": if user and hasattr(user, 'is_active') and user.is_active == "0":
return get_json_result( return get_json_result(
data=False, data=False,
code=settings.RetCode.FORBIDDEN, code=RetCode.FORBIDDEN,
message="This account has been disabled, please contact the administrator!", message="This account has been disabled, please contact the administrator!",
) )
elif user: elif user:
@ -128,7 +129,7 @@ def login():
else: else:
return get_json_result( return get_json_result(
data=False, data=False,
code=settings.RetCode.AUTHENTICATION_ERROR, code=RetCode.AUTHENTICATION_ERROR,
message="Email and password do not match!", message="Email and password do not match!",
) )
@ -151,7 +152,7 @@ def get_login_channels():
return get_json_result(data=channels) return get_json_result(data=channels)
except Exception as e: except Exception as e:
logging.exception(e) logging.exception(e)
return get_json_result(data=[], message=f"Load channels failure, error: {str(e)}", code=settings.RetCode.EXCEPTION_ERROR) return get_json_result(data=[], message=f"Load channels failure, error: {str(e)}", code=RetCode.EXCEPTION_ERROR)
@manager.route("/login/<channel>", methods=["GET"]) # noqa: F821 @manager.route("/login/<channel>", methods=["GET"]) # noqa: F821
@ -535,7 +536,7 @@ def setting_user():
if not check_password_hash(current_user.password, decrypt(request_data["password"])): if not check_password_hash(current_user.password, decrypt(request_data["password"])):
return get_json_result( return get_json_result(
data=False, data=False,
code=settings.RetCode.AUTHENTICATION_ERROR, code=RetCode.AUTHENTICATION_ERROR,
message="Password error!", message="Password error!",
) )
@ -563,7 +564,7 @@ def setting_user():
return get_json_result(data=True) return get_json_result(data=True)
except Exception as e: except Exception as e:
logging.exception(e) logging.exception(e)
return get_json_result(data=False, message="Update failure!", code=settings.RetCode.EXCEPTION_ERROR) return get_json_result(data=False, message="Update failure!", code=RetCode.EXCEPTION_ERROR)
@manager.route("/info", methods=["GET"]) # noqa: F821 @manager.route("/info", methods=["GET"]) # noqa: F821
@ -693,7 +694,7 @@ def user_add():
return get_json_result( return get_json_result(
data=False, data=False,
message="User registration is disabled!", message="User registration is disabled!",
code=settings.RetCode.OPERATING_ERROR, code=RetCode.OPERATING_ERROR,
) )
req = request.json req = request.json
@ -704,7 +705,7 @@ def user_add():
return get_json_result( return get_json_result(
data=False, data=False,
message=f"Invalid email address: {email_address}!", message=f"Invalid email address: {email_address}!",
code=settings.RetCode.OPERATING_ERROR, code=RetCode.OPERATING_ERROR,
) )
# Check if the email address is already used # Check if the email address is already used
@ -712,7 +713,7 @@ def user_add():
return get_json_result( return get_json_result(
data=False, data=False,
message=f"Email: {email_address} has already registered!", message=f"Email: {email_address} has already registered!",
code=settings.RetCode.OPERATING_ERROR, code=RetCode.OPERATING_ERROR,
) )
# Construct user info data # Construct user info data
@ -747,7 +748,7 @@ def user_add():
return get_json_result( return get_json_result(
data=False, data=False,
message=f"User registration failure, error: {str(e)}", message=f"User registration failure, error: {str(e)}",
code=settings.RetCode.EXCEPTION_ERROR, code=RetCode.EXCEPTION_ERROR,
) )
@ -847,11 +848,11 @@ def forget_get_captcha():
""" """
email = (request.args.get("email") or "") email = (request.args.get("email") or "")
if not email: if not email:
return get_json_result(data=False, code=settings.RetCode.ARGUMENT_ERROR, message="email is required") return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="email is required")
users = UserService.query(email=email) users = UserService.query(email=email)
if not users: if not users:
return get_json_result(data=False, code=settings.RetCode.DATA_ERROR, message="invalid email") return get_json_result(data=False, code=RetCode.DATA_ERROR, message="invalid email")
# Generate captcha text # Generate captcha text
allowed = string.ascii_uppercase + string.digits allowed = string.ascii_uppercase + string.digits
@ -878,17 +879,17 @@ def forget_send_otp():
captcha = (req.get("captcha") or "").strip() captcha = (req.get("captcha") or "").strip()
if not email or not captcha: if not email or not captcha:
return get_json_result(data=False, code=settings.RetCode.ARGUMENT_ERROR, message="email and captcha required") return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="email and captcha required")
users = UserService.query(email=email) users = UserService.query(email=email)
if not users: if not users:
return get_json_result(data=False, code=settings.RetCode.DATA_ERROR, message="invalid email") return get_json_result(data=False, code=RetCode.DATA_ERROR, message="invalid email")
stored_captcha = REDIS_CONN.get(captcha_key(email)) stored_captcha = REDIS_CONN.get(captcha_key(email))
if not stored_captcha: if not stored_captcha:
return get_json_result(data=False, code=settings.RetCode.NOT_EFFECTIVE, message="invalid or expired captcha") return get_json_result(data=False, code=RetCode.NOT_EFFECTIVE, message="invalid or expired captcha")
if (stored_captcha or "").strip().lower() != captcha.lower(): if (stored_captcha or "").strip().lower() != captcha.lower():
return get_json_result(data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="invalid or expired captcha") return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="invalid or expired captcha")
# Delete captcha to prevent reuse # Delete captcha to prevent reuse
REDIS_CONN.delete(captcha_key(email)) REDIS_CONN.delete(captcha_key(email))
@ -903,7 +904,7 @@ def forget_send_otp():
elapsed = RESEND_COOLDOWN_SECONDS elapsed = RESEND_COOLDOWN_SECONDS
remaining = RESEND_COOLDOWN_SECONDS - elapsed remaining = RESEND_COOLDOWN_SECONDS - elapsed
if remaining > 0: if remaining > 0:
return get_json_result(data=False, code=settings.RetCode.NOT_EFFECTIVE, message=f"you still have to wait {remaining} seconds") return get_json_result(data=False, code=RetCode.NOT_EFFECTIVE, message=f"you still have to wait {remaining} seconds")
# Generate OTP (uppercase letters only) and store hashed # Generate OTP (uppercase letters only) and store hashed
otp = "".join(secrets.choice(string.ascii_uppercase) for _ in range(OTP_LENGTH)) otp = "".join(secrets.choice(string.ascii_uppercase) for _ in range(OTP_LENGTH))
@ -928,9 +929,9 @@ def forget_send_otp():
ttl_min=ttl_min, ttl_min=ttl_min,
) )
except Exception: except Exception:
return get_json_result(data=False, code=settings.RetCode.SERVER_ERROR, message="failed to send email") return get_json_result(data=False, code=RetCode.SERVER_ERROR, message="failed to send email")
return get_json_result(data=True, code=settings.RetCode.SUCCESS, message="verification passed, email sent") return get_json_result(data=True, code=RetCode.SUCCESS, message="verification passed, email sent")
@manager.route("/forget", methods=["POST"]) # noqa: F821 @manager.route("/forget", methods=["POST"]) # noqa: F821
@ -946,31 +947,31 @@ def forget():
new_pwd2 = req.get("confirm_new_password") new_pwd2 = req.get("confirm_new_password")
if not all([email, otp, new_pwd, new_pwd2]): if not all([email, otp, new_pwd, new_pwd2]):
return get_json_result(data=False, code=settings.RetCode.ARGUMENT_ERROR, message="email, otp and passwords are required") return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="email, otp and passwords are required")
# For reset, passwords are provided as-is (no decrypt needed) # For reset, passwords are provided as-is (no decrypt needed)
if new_pwd != new_pwd2: if new_pwd != new_pwd2:
return get_json_result(data=False, code=settings.RetCode.ARGUMENT_ERROR, message="passwords do not match") return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="passwords do not match")
users = UserService.query(email=email) users = UserService.query(email=email)
if not users: if not users:
return get_json_result(data=False, code=settings.RetCode.DATA_ERROR, message="invalid email") return get_json_result(data=False, code=RetCode.DATA_ERROR, message="invalid email")
user = users[0] user = users[0]
# Verify OTP from Redis # Verify OTP from Redis
k_code, k_attempts, k_last, k_lock = otp_keys(email) k_code, k_attempts, k_last, k_lock = otp_keys(email)
if REDIS_CONN.get(k_lock): if REDIS_CONN.get(k_lock):
return get_json_result(data=False, code=settings.RetCode.NOT_EFFECTIVE, message="too many attempts, try later") return get_json_result(data=False, code=RetCode.NOT_EFFECTIVE, message="too many attempts, try later")
stored = REDIS_CONN.get(k_code) stored = REDIS_CONN.get(k_code)
if not stored: if not stored:
return get_json_result(data=False, code=settings.RetCode.NOT_EFFECTIVE, message="expired otp") return get_json_result(data=False, code=RetCode.NOT_EFFECTIVE, message="expired otp")
try: try:
stored_hash, salt_hex = str(stored).split(":", 1) stored_hash, salt_hex = str(stored).split(":", 1)
salt = bytes.fromhex(salt_hex) salt = bytes.fromhex(salt_hex)
except Exception: except Exception:
return get_json_result(data=False, code=settings.RetCode.EXCEPTION_ERROR, message="otp storage corrupted") return get_json_result(data=False, code=RetCode.EXCEPTION_ERROR, message="otp storage corrupted")
# Case-insensitive verification: OTP generated uppercase # Case-insensitive verification: OTP generated uppercase
calc = hash_code(otp.upper(), salt) calc = hash_code(otp.upper(), salt)
@ -983,7 +984,7 @@ def forget():
REDIS_CONN.set(k_attempts, attempts, OTP_TTL_SECONDS) REDIS_CONN.set(k_attempts, attempts, OTP_TTL_SECONDS)
if attempts >= ATTEMPT_LIMIT: if attempts >= ATTEMPT_LIMIT:
REDIS_CONN.set(k_lock, int(time.time()), ATTEMPT_LOCK_SECONDS) REDIS_CONN.set(k_lock, int(time.time()), ATTEMPT_LOCK_SECONDS)
return get_json_result(data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="expired otp") return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="expired otp")
# Success: consume OTP and reset password # Success: consume OTP and reset password
REDIS_CONN.delete(k_code) REDIS_CONN.delete(k_code)
@ -995,7 +996,7 @@ def forget():
UserService.update_user_password(user.id, new_pwd) UserService.update_user_password(user.id, new_pwd)
except Exception as e: except Exception as e:
logging.exception(e) logging.exception(e)
return get_json_result(data=False, code=settings.RetCode.EXCEPTION_ERROR, message="failed to reset password") return get_json_result(data=False, code=RetCode.EXCEPTION_ERROR, message="failed to reset password")
# Auto login (reuse login flow) # Auto login (reuse login flow)
user.access_token = get_uuid() user.access_token = get_uuid()

36
api/settings.py
View File

@ -17,7 +17,6 @@ import json
import os import os
import secrets import secrets
from datetime import date from datetime import date
from enum import Enum, IntEnum
import rag.utils import rag.utils
import rag.utils.es_conn import rag.utils.es_conn
@ -205,41 +204,6 @@ def init_settings():
MAIL_FRONTEND_URL = SMTP_CONF.get("mail_frontend_url", "") MAIL_FRONTEND_URL = SMTP_CONF.get("mail_frontend_url", "")
class CustomEnum(Enum):
@classmethod
def valid(cls, value):
try:
cls(value)
return True
except BaseException:
return False
@classmethod
def values(cls):
return [member.value for member in cls.__members__.values()]
@classmethod
def names(cls):
return [member.name for member in cls.__members__.values()]
class RetCode(IntEnum, CustomEnum):
SUCCESS = 0
NOT_EFFECTIVE = 10
EXCEPTION_ERROR = 100
ARGUMENT_ERROR = 101
DATA_ERROR = 102
OPERATING_ERROR = 103
CONNECTION_ERROR = 105
RUNNING = 106
PERMISSION_ERROR = 108
AUTHENTICATION_ERROR = 109
UNAUTHORIZED = 401
SERVER_ERROR = 500
FORBIDDEN = 403
NOT_FOUND = 404
def _parse_model_entry(entry): def _parse_model_entry(entry):
if isinstance(entry, str): if isinstance(entry, str):
return {"name": entry, "factory": None, "api_key": None, "base_url": None} return {"name": entry, "factory": None, "api_key": None, "base_url": None}

62
api/utils/api_utils.py
View File

@ -27,7 +27,6 @@ import trio
from flask import ( from flask import (
Response, Response,
jsonify, jsonify,
make_response,
) )
from flask_login import current_user from flask_login import current_user
from flask import ( from flask import (
@ -41,6 +40,8 @@ from api.db.db_models import APIToken
from api.utils.json_encode import CustomJSONEncoder from api.utils.json_encode import CustomJSONEncoder
from rag.utils.mcp_tool_call_conn import MCPToolCallSession, close_multiple_mcp_toolcall_sessions from rag.utils.mcp_tool_call_conn import MCPToolCallSession, close_multiple_mcp_toolcall_sessions
from common.connection_utils import timeout from common.connection_utils import timeout
from common.contants import RetCode
requests.models.complexjson.dumps = functools.partial(json.dumps, cls=CustomJSONEncoder) requests.models.complexjson.dumps = functools.partial(json.dumps, cls=CustomJSONEncoder)
@ -70,7 +71,7 @@ def serialize_for_json(obj):
# Fallback: convert to string representation # Fallback: convert to string representation
return str(obj) return str(obj)
def get_data_error_result(code=settings.RetCode.DATA_ERROR, message="Sorry! Data missing!"): def get_data_error_result(code=RetCode.DATA_ERROR, message="Sorry! Data missing!"):
logging.exception(Exception(message)) logging.exception(Exception(message))
result_dict = {"code": code, "message": message} result_dict = {"code": code, "message": message}
response = {} response = {}
@ -87,21 +88,21 @@ def server_error_response(e):
try: try:
msg = repr(e).lower() msg = repr(e).lower()
if getattr(e, "code", None) == 401 or ("unauthorized" in msg) or ("401" in msg): if getattr(e, "code", None) == 401 or ("unauthorized" in msg) or ("401" in msg):
return get_json_result(code=settings.RetCode.UNAUTHORIZED, message=repr(e)) return get_json_result(code=RetCode.UNAUTHORIZED, message=repr(e))
except Exception as ex: except Exception as ex:
logging.warning(f"error checking authorization: {ex}") logging.warning(f"error checking authorization: {ex}")
if len(e.args) > 1: if len(e.args) > 1:
try: try:
serialized_data = serialize_for_json(e.args[1]) serialized_data = serialize_for_json(e.args[1])
return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message=repr(e.args[0]), data=serialized_data) return get_json_result(code=RetCode.EXCEPTION_ERROR, message=repr(e.args[0]), data=serialized_data)
except Exception: except Exception:
return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message=repr(e.args[0]), data=None) return get_json_result(code=RetCode.EXCEPTION_ERROR, message=repr(e.args[0]), data=None)
if repr(e).find("index_not_found_exception") >= 0: if repr(e).find("index_not_found_exception") >= 0:
return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, return get_json_result(code=RetCode.EXCEPTION_ERROR,
message="No chunk found, please upload file and parse it.") message="No chunk found, please upload file and parse it.")
return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message=repr(e)) return get_json_result(code=RetCode.EXCEPTION_ERROR, message=repr(e))
def validate_request(*args, **kwargs): def validate_request(*args, **kwargs):
@ -130,7 +131,7 @@ def validate_request(*args, **kwargs):
if error_arguments: if error_arguments:
error_string += "required argument values: {}".format( error_string += "required argument values: {}".format(
",".join(["{}={}".format(a[0], a[1]) for a in error_arguments])) ",".join(["{}={}".format(a[0], a[1]) for a in error_arguments]))
return get_json_result(code=settings.RetCode.ARGUMENT_ERROR, message=error_string) return get_json_result(code=RetCode.ARGUMENT_ERROR, message=error_string)
return func(*_args, **_kwargs) return func(*_args, **_kwargs)
return decorated_function return decorated_function
@ -144,7 +145,7 @@ def not_allowed_parameters(*params):
input_arguments = flask_request.json or flask_request.form.to_dict() input_arguments = flask_request.json or flask_request.form.to_dict()
for param in params: for param in params:
if param in input_arguments: if param in input_arguments:
return get_json_result(code=settings.RetCode.ARGUMENT_ERROR, return get_json_result(code=RetCode.ARGUMENT_ERROR,
message=f"Parameter {param} isn't allowed") message=f"Parameter {param} isn't allowed")
return f(*args, **kwargs) return f(*args, **kwargs)
@ -161,13 +162,13 @@ def active_required(f):
usr = UserService.filter_by_id(user_id) usr = UserService.filter_by_id(user_id)
# check is_active # check is_active
if not usr or not usr.is_active == ActiveEnum.ACTIVE.value: if not usr or not usr.is_active == ActiveEnum.ACTIVE.value:
return get_json_result(code=settings.RetCode.FORBIDDEN, message="User isn't active, please activate first.") return get_json_result(code=RetCode.FORBIDDEN, message="User isn't active, please activate first.")
return f(*args, **kwargs) return f(*args, **kwargs)
return wrapper return wrapper
def get_json_result(code: settings.RetCode = settings.RetCode.SUCCESS, message="success", data=None): def get_json_result(code: RetCode = RetCode.SUCCESS, message="success", data=None):
response = {"code": code, "message": message, "data": data} response = {"code": code, "message": message, "data": data}
return jsonify(response) return jsonify(response)
@ -178,40 +179,21 @@ def apikey_required(func):
token = flask_request.headers.get("Authorization").split()[1] token = flask_request.headers.get("Authorization").split()[1]
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return build_error_result(message="API-KEY is invalid!", code=settings.RetCode.FORBIDDEN) return build_error_result(message="API-KEY is invalid!", code=RetCode.FORBIDDEN)
kwargs["tenant_id"] = objs[0].tenant_id kwargs["tenant_id"] = objs[0].tenant_id
return func(*args, **kwargs) return func(*args, **kwargs)
return decorated_function return decorated_function
def build_error_result(code=settings.RetCode.FORBIDDEN, message="success"): def build_error_result(code=RetCode.FORBIDDEN, message="success"):
response = {"code": code, "message": message} response = {"code": code, "message": message}
response = jsonify(response) response = jsonify(response)
response.status_code = code response.status_code = code
return response return response
def construct_response(code=settings.RetCode.SUCCESS, message="success", data=None, auth=None): def construct_json_result(code: RetCode = RetCode.SUCCESS, message="success", data=None):
result_dict = {"code": code, "message": message, "data": data}
response_dict = {}
for key, value in result_dict.items():
if value is None and key != "code":
continue
else:
response_dict[key] = value
response = make_response(jsonify(response_dict))
if auth:
response.headers["Authorization"] = auth
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Method"] = "*"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Expose-Headers"] = "Authorization"
return response
def construct_json_result(code: settings.RetCode = settings.RetCode.SUCCESS, message="success", data=None):
if data is None: if data is None:
return jsonify({"code": code, "message": message}) return jsonify({"code": code, "message": message})
else: else:
@ -232,14 +214,14 @@ def token_required(func):
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result(data=False, message="Authentication error: API key is invalid!", return get_json_result(data=False, message="Authentication error: API key is invalid!",
code=settings.RetCode.AUTHENTICATION_ERROR) code=RetCode.AUTHENTICATION_ERROR)
kwargs["tenant_id"] = objs[0].tenant_id kwargs["tenant_id"] = objs[0].tenant_id
return func(*args, **kwargs) return func(*args, **kwargs)
return decorated_function return decorated_function
def get_result(code=settings.RetCode.SUCCESS, message="", data=None, total=None): def get_result(code=RetCode.SUCCESS, message="", data=None, total=None):
""" """
Standard API response format: Standard API response format:
{ {
@ -251,7 +233,7 @@ def get_result(code=settings.RetCode.SUCCESS, message="", data=None, total=None)
""" """
response = {"code": code} response = {"code": code}
if code == settings.RetCode.SUCCESS: if code == RetCode.SUCCESS:
if data is not None: if data is not None:
response["data"] = data response["data"] = data
if total is not None: if total is not None:
@ -263,7 +245,7 @@ def get_result(code=settings.RetCode.SUCCESS, message="", data=None, total=None)
def get_error_data_result( def get_error_data_result(
message="Sorry! Data missing!", message="Sorry! Data missing!",
code=settings.RetCode.DATA_ERROR, code=RetCode.DATA_ERROR,
): ):
result_dict = {"code": code, "message": message} result_dict = {"code": code, "message": message}
response = {} response = {}
@ -276,15 +258,15 @@ def get_error_data_result(
def get_error_argument_result(message="Invalid arguments"): def get_error_argument_result(message="Invalid arguments"):
return get_result(code=settings.RetCode.ARGUMENT_ERROR, message=message) return get_result(code=RetCode.ARGUMENT_ERROR, message=message)
def get_error_permission_result(message="Permission error"): def get_error_permission_result(message="Permission error"):
return get_result(code=settings.RetCode.PERMISSION_ERROR, message=message) return get_result(code=RetCode.PERMISSION_ERROR, message=message)
def get_error_operating_result(message="Operating error"): def get_error_operating_result(message="Operating error"):
return get_result(code=settings.RetCode.OPERATING_ERROR, message=message) return get_result(code=RetCode.OPERATING_ERROR, message=message)
def generate_confirmation_token(): def generate_confirmation_token():

21
common/connection_utils.py
View File

@ -21,6 +21,8 @@ from typing import Any, Callable, Coroutine, Optional, Type, Union
import asyncio import asyncio
import trio import trio
from functools import wraps from functools import wraps
from flask import make_response, jsonify
from common.contants import RetCode
TimeoutException = Union[Type[BaseException], BaseException] TimeoutException = Union[Type[BaseException], BaseException]
OnTimeoutCallback = Union[Callable[..., Any], Coroutine[Any, Any, Any]] OnTimeoutCallback = Union[Callable[..., Any], Coroutine[Any, Any, Any]]
@ -99,3 +101,22 @@ def timeout(seconds: float | int | str = None, attempts: int = 2, *, exception:
return wrapper return wrapper
return decorator return decorator
def construct_response(code=RetCode.SUCCESS, message="success", data=None, auth=None):
result_dict = {"code": code, "message": message, "data": data}
response_dict = {}
for key, value in result_dict.items():
if value is None and key != "code":
continue
else:
response_dict[key] = value
response = make_response(jsonify(response_dict))
if auth:
response.headers["Authorization"] = auth
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Method"] = "*"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Expose-Headers"] = "Authorization"
return response

39
common/contants.py
View File

@ -14,8 +14,45 @@
# limitations under the License. # limitations under the License.
# #
from enum import Enum, IntEnum
SERVICE_CONF = "service_conf.yaml" SERVICE_CONF = "service_conf.yaml"
class CustomEnum(Enum):
@classmethod
def valid(cls, value):
try:
cls(value)
return True
except BaseException:
return False
@classmethod
def values(cls):
return [member.value for member in cls.__members__.values()]
@classmethod
def names(cls):
return [member.name for member in cls.__members__.values()]
class RetCode(IntEnum, CustomEnum):
SUCCESS = 0
NOT_EFFECTIVE = 10
EXCEPTION_ERROR = 100
ARGUMENT_ERROR = 101
DATA_ERROR = 102
OPERATING_ERROR = 103
CONNECTION_ERROR = 105
RUNNING = 106
PERMISSION_ERROR = 108
AUTHENTICATION_ERROR = 109
UNAUTHORIZED = 401
SERVER_ERROR = 500
FORBIDDEN = 403
NOT_FOUND = 404
# environment # environment
# ENV_STRONG_TEST_COUNT = "STRONG_TEST_COUNT" # ENV_STRONG_TEST_COUNT = "STRONG_TEST_COUNT"
# ENV_RAGFLOW_SECRET_KEY = "RAGFLOW_SECRET_KEY" # ENV_RAGFLOW_SECRET_KEY = "RAGFLOW_SECRET_KEY"
@ -60,5 +97,3 @@ SERVICE_CONF = "service_conf.yaml"
# ENV_MAX_CONCURRENT_MINIO = "MAX_CONCURRENT_MINIO" # ENV_MAX_CONCURRENT_MINIO = "MAX_CONCURRENT_MINIO"
# ENV_WORKER_HEARTBEAT_TIMEOUT = "WORKER_HEARTBEAT_TIMEOUT" # ENV_WORKER_HEARTBEAT_TIMEOUT = "WORKER_HEARTBEAT_TIMEOUT"
# ENV_TRACE_MALLOC_ENABLED = "TRACE_MALLOC_ENABLED" # ENV_TRACE_MALLOC_ENABLED = "TRACE_MALLOC_ENABLED"