mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-12-08 20:42:30 +08:00
Refactor API for document and session (#2819)
### What problem does this PR solve? Refactor API for document and session. ### Type of change - [x] Refactoring --------- Co-authored-by: liuhua <10215101452@stu.ecun.edu.cn>
This commit is contained in:
liuhua
@ -4,9 +4,11 @@ import datetime
|
||||
import json
|
||||
import traceback
|
||||
|
||||
from botocore.docs.method import document_model_driven_method
|
||||
from flask import request
|
||||
from flask_login import login_required, current_user
|
||||
from elasticsearch_dsl import Q
|
||||
from sphinx.addnodes import document
|
||||
|
||||
from rag.app.qa import rmPrefix, beAdoc
|
||||
from rag.nlp import search, rag_tokenizer, keyword_extraction
|
||||
@ -16,22 +18,22 @@ from api.db import LLMType, ParserType
|
||||
from api.db.services.knowledgebase_service import KnowledgebaseService
|
||||
from api.db.services.llm_service import TenantLLMService
|
||||
from api.db.services.user_service import UserTenantService
|
||||
from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
|
||||
from api.utils.api_utils import server_error_response, get_error_data_result, validate_request
|
||||
from api.db.services.document_service import DocumentService
|
||||
from api.settings import RetCode, retrievaler, kg_retrievaler
|
||||
from api.utils.api_utils import get_json_result
|
||||
from api.utils.api_utils import get_result
|
||||
import hashlib
|
||||
import re
|
||||
from api.utils.api_utils import get_json_result, token_required, get_data_error_result
|
||||
from api.utils.api_utils import get_result, token_required, get_error_data_result
|
||||
|
||||
from api.db.db_models import Task, File
|
||||
|
||||
from api.db.services.task_service import TaskService, queue_tasks
|
||||
from api.db.services.user_service import TenantService, UserTenantService
|
||||
|
||||
from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
|
||||
from api.utils.api_utils import server_error_response, get_error_data_result, validate_request
|
||||
|
||||
from api.utils.api_utils import get_json_result
|
||||
from api.utils.api_utils import get_result, get_result, get_error_data_result
|
||||
|
||||
from functools import partial
|
||||
from io import BytesIO
|
||||
@ -59,307 +61,163 @@ MAXIMUM_OF_UPLOADING_FILES = 256
|
||||
MAXIMUM_OF_UPLOADING_FILES = 256
|
||||
|
||||
|
||||
@manager.route('/dataset/<dataset_id>/documents/upload', methods=['POST'])
|
||||
@manager.route('/dataset/<dataset_id>/document', methods=['POST'])
|
||||
@token_required
|
||||
def upload(dataset_id, tenant_id):
|
||||
if 'file' not in request.files:
|
||||
return get_json_result(
|
||||
data=False, retmsg='No file part!', retcode=RetCode.ARGUMENT_ERROR)
|
||||
return get_error_data_result(
|
||||
retmsg='No file part!', retcode=RetCode.ARGUMENT_ERROR)
|
||||
file_objs = request.files.getlist('file')
|
||||
for file_obj in file_objs:
|
||||
if file_obj.filename == '':
|
||||
return get_json_result(
|
||||
data=False, retmsg='No file selected!', retcode=RetCode.ARGUMENT_ERROR)
|
||||
return get_result(
|
||||
retmsg='No file selected!', retcode=RetCode.ARGUMENT_ERROR)
|
||||
e, kb = KnowledgebaseService.get_by_id(dataset_id)
|
||||
if not e:
|
||||
raise LookupError(f"Can't find the knowledgebase with ID {dataset_id}!")
|
||||
err, _ = FileService.upload_document(kb, file_objs, tenant_id)
|
||||
if err:
|
||||
return get_json_result(
|
||||
data=False, retmsg="\n".join(err), retcode=RetCode.SERVER_ERROR)
|
||||
return get_json_result(data=True)
|
||||
return get_result(
|
||||
retmsg="\n".join(err), retcode=RetCode.SERVER_ERROR)
|
||||
return get_result()
|
||||
|
||||
|
||||
@manager.route('/infos', methods=['GET'])
|
||||
@manager.route('/dataset/<dataset_id>/info/<document_id>', methods=['PUT'])
|
||||
@token_required
|
||||
def docinfos(tenant_id):
|
||||
req = request.args
|
||||
if "id" not in req and "name" not in req:
|
||||
return get_data_error_result(
|
||||
retmsg="Id or name should be provided")
|
||||
doc_id=None
|
||||
if "id" in req:
|
||||
doc_id = req["id"]
|
||||
if "name" in req:
|
||||
doc_name = req["name"]
|
||||
doc_id = DocumentService.get_doc_id_by_doc_name(doc_name)
|
||||
e, doc = DocumentService.get_by_id(doc_id)
|
||||
#rename key's name
|
||||
key_mapping = {
|
||||
"chunk_num": "chunk_count",
|
||||
"kb_id": "knowledgebase_id",
|
||||
"token_num": "token_count",
|
||||
"parser_id":"parser_method",
|
||||
}
|
||||
renamed_doc = {}
|
||||
for key, value in doc.to_dict().items():
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_doc[new_key] = value
|
||||
|
||||
return get_json_result(data=renamed_doc)
|
||||
|
||||
|
||||
@manager.route('/save', methods=['POST'])
|
||||
@token_required
|
||||
def save_doc(tenant_id):
|
||||
def update_doc(tenant_id, dataset_id, document_id):
|
||||
req = request.json
|
||||
#get doc by id or name
|
||||
doc_id = None
|
||||
if "id" in req:
|
||||
doc_id = req["id"]
|
||||
elif "name" in req:
|
||||
doc_name = req["name"]
|
||||
doc_id = DocumentService.get_doc_id_by_doc_name(doc_name)
|
||||
if not doc_id:
|
||||
return get_json_result(retcode=400, retmsg="Document ID or name is required")
|
||||
e, doc = DocumentService.get_by_id(doc_id)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
#other value can't be changed
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg='You do not own the dataset.')
|
||||
doc = DocumentService.query(kb_id=dataset_id, id=document_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg='The dataset not own the document.')
|
||||
doc = doc[0]
|
||||
if "chunk_count" in req:
|
||||
if req["chunk_count"] != doc.chunk_num:
|
||||
return get_data_error_result(
|
||||
retmsg="Can't change chunk_count.")
|
||||
return get_error_data_result(retmsg="Can't change chunk_count.")
|
||||
if "token_count" in req:
|
||||
if req["token_count"] != doc.token_num:
|
||||
return get_data_error_result(
|
||||
retmsg="Can't change token_count.")
|
||||
return get_error_data_result(retmsg="Can't change token_count.")
|
||||
if "progress" in req:
|
||||
if req['progress'] != doc.progress:
|
||||
return get_data_error_result(
|
||||
retmsg="Can't change progress.")
|
||||
#change name or parse_method
|
||||
return get_error_data_result(retmsg="Can't change progress.")
|
||||
|
||||
if "name" in req and req["name"] != doc.name:
|
||||
try:
|
||||
if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
|
||||
doc.name.lower()).suffix:
|
||||
return get_json_result(
|
||||
data=False,
|
||||
retmsg="The extension of file can't be changed",
|
||||
retcode=RetCode.ARGUMENT_ERROR)
|
||||
for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
|
||||
if d.name == req["name"]:
|
||||
return get_data_error_result(
|
||||
retmsg="Duplicated document name in the same knowledgebase.")
|
||||
if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
|
||||
return get_result(retmsg="The extension of file can't be changed", retcode=RetCode.ARGUMENT_ERROR)
|
||||
for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
|
||||
if d.name == req["name"]:
|
||||
return get_error_data_result(
|
||||
retmsg="Duplicated document name in the same knowledgebase.")
|
||||
if not DocumentService.update_by_id(
|
||||
document_id, {"name": req["name"]}):
|
||||
return get_error_data_result(
|
||||
retmsg="Database error (Document rename)!")
|
||||
|
||||
if not DocumentService.update_by_id(
|
||||
doc_id, {"name": req["name"]}):
|
||||
return get_data_error_result(
|
||||
retmsg="Database error (Document rename)!")
|
||||
|
||||
informs = File2DocumentService.get_by_document_id(doc_id)
|
||||
if informs:
|
||||
e, file = FileService.get_by_id(informs[0].file_id)
|
||||
FileService.update_by_id(file.id, {"name": req["name"]})
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
informs = File2DocumentService.get_by_document_id(document_id)
|
||||
if informs:
|
||||
e, file = FileService.get_by_id(informs[0].file_id)
|
||||
FileService.update_by_id(file.id, {"name": req["name"]})
|
||||
if "parser_method" in req:
|
||||
try:
|
||||
if doc.parser_id.lower() == req["parser_method"].lower():
|
||||
if "parser_config" in req:
|
||||
if req["parser_config"] == doc.parser_config:
|
||||
return get_json_result(data=True)
|
||||
else:
|
||||
return get_json_result(data=True)
|
||||
|
||||
if doc.type == FileType.VISUAL or re.search(
|
||||
r"\.(ppt|pptx|pages)$", doc.name):
|
||||
return get_data_error_result(retmsg="Not supported yet!")
|
||||
|
||||
e = DocumentService.update_by_id(doc.id,
|
||||
{"parser_id": req["parser_method"], "progress": 0, "progress_msg": "",
|
||||
"run": TaskStatus.UNSTART.value})
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
if "parser_config" in req:
|
||||
DocumentService.update_parser_config(doc.id, req["parser_config"])
|
||||
if doc.token_num > 0:
|
||||
e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1,
|
||||
doc.process_duation * -1)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
tenant_id = DocumentService.get_tenant_id(req["id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
ELASTICSEARCH.deleteByQuery(
|
||||
Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
return get_json_result(data=True)
|
||||
|
||||
|
||||
|
||||
@manager.route('/change_parser', methods=['POST'])
|
||||
@token_required
|
||||
def change_parser(tenant_id):
|
||||
req = request.json
|
||||
try:
|
||||
e, doc = DocumentService.get_by_id(req["doc_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
if doc.parser_id.lower() == req["parser_id"].lower():
|
||||
if doc.parser_id.lower() == req["parser_method"].lower():
|
||||
if "parser_config" in req:
|
||||
if req["parser_config"] == doc.parser_config:
|
||||
return get_json_result(data=True)
|
||||
return get_result(retcode=RetCode.SUCCESS)
|
||||
else:
|
||||
return get_json_result(data=True)
|
||||
return get_result(retcode=RetCode.SUCCESS)
|
||||
|
||||
if doc.type == FileType.VISUAL or re.search(
|
||||
r"\.(ppt|pptx|pages)$", doc.name):
|
||||
return get_data_error_result(retmsg="Not supported yet!")
|
||||
return get_error_data_result(retmsg="Not supported yet!")
|
||||
|
||||
e = DocumentService.update_by_id(doc.id,
|
||||
{"parser_id": req["parser_id"], "progress": 0, "progress_msg": "",
|
||||
{"parser_id": req["parser_method"], "progress": 0, "progress_msg": "",
|
||||
"run": TaskStatus.UNSTART.value})
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
if "parser_config" in req:
|
||||
DocumentService.update_parser_config(doc.id, req["parser_config"])
|
||||
return get_error_data_result(retmsg="Document not found!")
|
||||
if doc.token_num > 0:
|
||||
e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1,
|
||||
doc.process_duation * -1)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
tenant_id = DocumentService.get_tenant_id(req["doc_id"])
|
||||
return get_error_data_result(retmsg="Document not found!")
|
||||
tenant_id = DocumentService.get_tenant_id(req["id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
return get_error_data_result(retmsg="Tenant not found!")
|
||||
ELASTICSEARCH.deleteByQuery(
|
||||
Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
|
||||
if "parser_config" in req:
|
||||
DocumentService.update_parser_config(doc.id, req["parser_config"])
|
||||
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
|
||||
@manager.route('/rename', methods=['POST'])
|
||||
@login_required
|
||||
@validate_request("doc_id", "name")
|
||||
def rename():
|
||||
req = request.json
|
||||
try:
|
||||
e, doc = DocumentService.get_by_id(req["doc_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
|
||||
doc.name.lower()).suffix:
|
||||
return get_json_result(
|
||||
data=False,
|
||||
retmsg="The extension of file can't be changed",
|
||||
retcode=RetCode.ARGUMENT_ERROR)
|
||||
for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
|
||||
if d.name == req["name"]:
|
||||
return get_data_error_result(
|
||||
retmsg="Duplicated document name in the same knowledgebase.")
|
||||
|
||||
if not DocumentService.update_by_id(
|
||||
req["doc_id"], {"name": req["name"]}):
|
||||
return get_data_error_result(
|
||||
retmsg="Database error (Document rename)!")
|
||||
|
||||
informs = File2DocumentService.get_by_document_id(req["doc_id"])
|
||||
if informs:
|
||||
e, file = FileService.get_by_id(informs[0].file_id)
|
||||
FileService.update_by_id(file.id, {"name": req["name"]})
|
||||
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
return get_result()
|
||||
|
||||
|
||||
@manager.route("/<document_id>", methods=["GET"])
|
||||
@manager.route('/dataset/<dataset_id>/document/<document_id>', methods=['GET'])
|
||||
@token_required
|
||||
def download_document(document_id,tenant_id):
|
||||
try:
|
||||
# Check whether there is this document
|
||||
exist, document = DocumentService.get_by_id(document_id)
|
||||
if not exist:
|
||||
return construct_json_result(message=f"This document '{document_id}' cannot be found!",
|
||||
code=RetCode.ARGUMENT_ERROR)
|
||||
|
||||
# The process of downloading
|
||||
doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id) # minio address
|
||||
file_stream = STORAGE_IMPL.get(doc_id, doc_location)
|
||||
if not file_stream:
|
||||
return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR)
|
||||
|
||||
file = BytesIO(file_stream)
|
||||
|
||||
# Use send_file with a proper filename and MIME type
|
||||
return send_file(
|
||||
file,
|
||||
as_attachment=True,
|
||||
download_name=document.name,
|
||||
mimetype='application/octet-stream' # Set a default MIME type
|
||||
)
|
||||
|
||||
# Error
|
||||
except Exception as e:
|
||||
return construct_error_response(e)
|
||||
def download(tenant_id, dataset_id, document_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f'You do not own the dataset {dataset_id}.')
|
||||
doc = DocumentService.query(kb_id=dataset_id, id=document_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f'The dataset not own the document {doc.id}.')
|
||||
# The process of downloading
|
||||
doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id) # minio address
|
||||
file_stream = STORAGE_IMPL.get(doc_id, doc_location)
|
||||
if not file_stream:
|
||||
return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR)
|
||||
file = BytesIO(file_stream)
|
||||
# Use send_file with a proper filename and MIME type
|
||||
return send_file(
|
||||
file,
|
||||
as_attachment=True,
|
||||
download_name=doc[0].name,
|
||||
mimetype='application/octet-stream' # Set a default MIME type
|
||||
)
|
||||
|
||||
|
||||
@manager.route('/dataset/<dataset_id>/documents', methods=['GET'])
|
||||
@manager.route('/dataset/<dataset_id>/info', methods=['GET'])
|
||||
@token_required
|
||||
def list_docs(dataset_id, tenant_id):
|
||||
kb_id = request.args.get("knowledgebase_id")
|
||||
if not kb_id:
|
||||
return get_json_result(
|
||||
data=False, retmsg='Lack of "KB ID"', retcode=RetCode.ARGUMENT_ERROR)
|
||||
tenants = UserTenantService.query(user_id=tenant_id)
|
||||
for tenant in tenants:
|
||||
if KnowledgebaseService.query(
|
||||
tenant_id=tenant.tenant_id, id=kb_id):
|
||||
break
|
||||
else:
|
||||
return get_json_result(
|
||||
data=False, retmsg=f'Only owner of knowledgebase authorized for this operation.',
|
||||
retcode=RetCode.OPERATING_ERROR)
|
||||
keywords = request.args.get("keywords", "")
|
||||
|
||||
page_number = int(request.args.get("page", 1))
|
||||
items_per_page = int(request.args.get("page_size", 15))
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}. ")
|
||||
id = request.args.get("id")
|
||||
if not DocumentService.query(id=id,kb_id=dataset_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the document {id}.")
|
||||
offset = int(request.args.get("offset", 1))
|
||||
keywords = request.args.get("keywords","")
|
||||
limit = int(request.args.get("limit", 1024))
|
||||
orderby = request.args.get("orderby", "create_time")
|
||||
desc = request.args.get("desc", True)
|
||||
try:
|
||||
docs, tol = DocumentService.get_by_kb_id(
|
||||
kb_id, page_number, items_per_page, orderby, desc, keywords)
|
||||
if request.args.get("desc") == "False":
|
||||
desc = False
|
||||
else:
|
||||
desc = True
|
||||
docs, tol = DocumentService.get_list(dataset_id, offset, limit, orderby, desc, keywords, id)
|
||||
|
||||
# rename key's name
|
||||
renamed_doc_list = []
|
||||
for doc in docs:
|
||||
key_mapping = {
|
||||
"chunk_num": "chunk_count",
|
||||
"kb_id": "knowledgebase_id",
|
||||
"token_num": "token_count",
|
||||
"parser_id":"parser_method"
|
||||
}
|
||||
renamed_doc = {}
|
||||
for key, value in doc.items():
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_doc[new_key] = value
|
||||
renamed_doc_list.append(renamed_doc)
|
||||
return get_json_result(data={"total": tol, "docs": renamed_doc_list})
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
# rename key's name
|
||||
renamed_doc_list = []
|
||||
for doc in docs:
|
||||
key_mapping = {
|
||||
"chunk_num": "chunk_count",
|
||||
"kb_id": "knowledgebase_id",
|
||||
"token_num": "token_count",
|
||||
"parser_id": "parser_method"
|
||||
}
|
||||
renamed_doc = {}
|
||||
for key, value in doc.items():
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_doc[new_key] = value
|
||||
renamed_doc_list.append(renamed_doc)
|
||||
return get_result(data={"total": tol, "docs": renamed_doc_list})
|
||||
|
||||
|
||||
@manager.route('/delete', methods=['DELETE'])
|
||||
@manager.route('/dataset/<dataset_id>/document', methods=['DELETE'])
|
||||
@token_required
|
||||
def rm(tenant_id):
|
||||
req = request.args
|
||||
if "document_id" not in req:
|
||||
return get_data_error_result(
|
||||
retmsg="doc_id is required")
|
||||
doc_ids = req["document_id"]
|
||||
if isinstance(doc_ids, str): doc_ids = [doc_ids]
|
||||
def delete(tenant_id,dataset_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}. ")
|
||||
req = request.json
|
||||
if not req.get("ids"):
|
||||
return get_error_data_result(retmsg="ids is required")
|
||||
doc_ids = req["ids"]
|
||||
root_folder = FileService.get_root_folder(tenant_id)
|
||||
pf_id = root_folder["id"]
|
||||
FileService.init_knowledgebase_docs(pf_id, tenant_id)
|
||||
@ -368,15 +226,15 @@ def rm(tenant_id):
|
||||
try:
|
||||
e, doc = DocumentService.get_by_id(doc_id)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
return get_error_data_result(retmsg="Document not found!")
|
||||
tenant_id = DocumentService.get_tenant_id(doc_id)
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
return get_error_data_result(retmsg="Tenant not found!")
|
||||
|
||||
b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
|
||||
|
||||
if not DocumentService.remove_document(doc, tenant_id):
|
||||
return get_data_error_result(
|
||||
return get_error_data_result(
|
||||
retmsg="Database error (Document removal)!")
|
||||
|
||||
f2d = File2DocumentService.get_by_document_id(doc_id)
|
||||
@ -388,80 +246,69 @@ def rm(tenant_id):
|
||||
errors += str(e)
|
||||
|
||||
if errors:
|
||||
return get_json_result(data=False, retmsg=errors, retcode=RetCode.SERVER_ERROR)
|
||||
return get_result(retmsg=errors, retcode=RetCode.SERVER_ERROR)
|
||||
|
||||
return get_json_result(data=True, retmsg="success")
|
||||
return get_result()
|
||||
|
||||
@manager.route("/<document_id>/status", methods=["GET"])
|
||||
|
||||
@manager.route('/dataset/<dataset_id>/chunk', methods=['POST'])
|
||||
@token_required
|
||||
def show_parsing_status(tenant_id, document_id):
|
||||
try:
|
||||
# valid document
|
||||
exist, _ = DocumentService.get_by_id(document_id)
|
||||
if not exist:
|
||||
return construct_json_result(code=RetCode.DATA_ERROR,
|
||||
message=f"This document: '{document_id}' is not a valid document.")
|
||||
|
||||
_, doc = DocumentService.get_by_id(document_id) # get doc object
|
||||
doc_attributes = doc.to_dict()
|
||||
|
||||
return construct_json_result(
|
||||
data={"progress": doc_attributes["progress"], "status": TaskStatus(doc_attributes["status"]).name},
|
||||
code=RetCode.SUCCESS
|
||||
)
|
||||
except Exception as e:
|
||||
return construct_error_response(e)
|
||||
|
||||
|
||||
|
||||
@manager.route('/run', methods=['POST'])
|
||||
@token_required
|
||||
def run(tenant_id):
|
||||
def parse(tenant_id,dataset_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
req = request.json
|
||||
try:
|
||||
for id in req["document_ids"]:
|
||||
info = {"run": str(req["run"]), "progress": 0}
|
||||
if str(req["run"]) == TaskStatus.RUNNING.value:
|
||||
info["progress_msg"] = ""
|
||||
info["chunk_num"] = 0
|
||||
info["token_num"] = 0
|
||||
DocumentService.update_by_id(id, info)
|
||||
# if str(req["run"]) == TaskStatus.CANCEL.value:
|
||||
tenant_id = DocumentService.get_tenant_id(id)
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
ELASTICSEARCH.deleteByQuery(
|
||||
Q("match", doc_id=id), idxnm=search.index_name(tenant_id))
|
||||
for id in req["document_ids"]:
|
||||
if not DocumentService.query(id=id,kb_id=dataset_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the document {id}.")
|
||||
info = {"run": "1", "progress": 0}
|
||||
info["progress_msg"] = ""
|
||||
info["chunk_num"] = 0
|
||||
info["token_num"] = 0
|
||||
DocumentService.update_by_id(id, info)
|
||||
# if str(req["run"]) == TaskStatus.CANCEL.value:
|
||||
ELASTICSEARCH.deleteByQuery(
|
||||
Q("match", doc_id=id), idxnm=search.index_name(tenant_id))
|
||||
TaskService.filter_delete([Task.doc_id == id])
|
||||
e, doc = DocumentService.get_by_id(id)
|
||||
doc = doc.to_dict()
|
||||
doc["tenant_id"] = tenant_id
|
||||
bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
|
||||
queue_tasks(doc, bucket, name)
|
||||
return get_result()
|
||||
|
||||
if str(req["run"]) == TaskStatus.RUNNING.value:
|
||||
TaskService.filter_delete([Task.doc_id == id])
|
||||
e, doc = DocumentService.get_by_id(id)
|
||||
doc = doc.to_dict()
|
||||
doc["tenant_id"] = tenant_id
|
||||
bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
|
||||
queue_tasks(doc, bucket, name)
|
||||
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
|
||||
|
||||
@manager.route('/chunk/list', methods=['POST'])
|
||||
@manager.route('/dataset/<dataset_id>/chunk', methods=['DELETE'])
|
||||
@token_required
|
||||
@validate_request("document_id")
|
||||
def list_chunk(tenant_id):
|
||||
def stop_parsing(tenant_id,dataset_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
req = request.json
|
||||
doc_id = req["document_id"]
|
||||
page = int(req.get("page", 1))
|
||||
size = int(req.get("size", 30))
|
||||
for id in req["document_ids"]:
|
||||
if not DocumentService.query(id=id,kb_id=dataset_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the document {id}.")
|
||||
info = {"run": "2", "progress": 0}
|
||||
DocumentService.update_by_id(id, info)
|
||||
# if str(req["run"]) == TaskStatus.CANCEL.value:
|
||||
tenant_id = DocumentService.get_tenant_id(id)
|
||||
ELASTICSEARCH.deleteByQuery(
|
||||
Q("match", doc_id=id), idxnm=search.index_name(tenant_id))
|
||||
return get_result()
|
||||
|
||||
|
||||
@manager.route('/dataset/{dataset_id}/document/{document_id}/chunk', methods=['GET'])
|
||||
@token_required
|
||||
def list_chunk(tenant_id,dataset_id,document_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
doc=DocumentService.query(id=document_id, kb_id=dataset_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f"You don't own the document {document_id}.")
|
||||
doc=doc[0]
|
||||
req = request.args
|
||||
doc_id = document_id
|
||||
page = int(req.get("offset", 1))
|
||||
size = int(req.get("limit", 30))
|
||||
question = req.get("keywords", "")
|
||||
try:
|
||||
tenant_id = DocumentService.get_tenant_id(req["document_id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
e, doc = DocumentService.get_by_id(doc_id)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
query = {
|
||||
"doc_ids": [doc_id], "page": page, "size": size, "question": question, "sort": True
|
||||
}
|
||||
@ -470,7 +317,7 @@ def list_chunk(tenant_id):
|
||||
sres = retrievaler.search(query, search.index_name(tenant_id), highlight=True)
|
||||
res = {"total": sres.total, "chunks": [], "doc": doc.to_dict()}
|
||||
|
||||
origin_chunks=[]
|
||||
origin_chunks = []
|
||||
for id in sres.ids:
|
||||
d = {
|
||||
"chunk_id": id,
|
||||
@ -490,7 +337,7 @@ def list_chunk(tenant_id):
|
||||
poss.append([float(d["positions"][i]), float(d["positions"][i + 1]), float(d["positions"][i + 2]),
|
||||
float(d["positions"][i + 3]), float(d["positions"][i + 4])])
|
||||
d["positions"] = poss
|
||||
|
||||
|
||||
origin_chunks.append(d)
|
||||
##rename keys
|
||||
for chunk in origin_chunks:
|
||||
@ -499,28 +346,34 @@ def list_chunk(tenant_id):
|
||||
"content_with_weight": "content",
|
||||
"doc_id": "document_id",
|
||||
"important_kwd": "important_keywords",
|
||||
"img_id":"image_id",
|
||||
"img_id": "image_id",
|
||||
}
|
||||
renamed_chunk = {}
|
||||
for key, value in chunk.items():
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_chunk[new_key] = value
|
||||
res["chunks"].append(renamed_chunk)
|
||||
return get_json_result(data=res)
|
||||
return get_result(data=res)
|
||||
except Exception as e:
|
||||
if str(e).find("not_found") > 0:
|
||||
return get_json_result(data=False, retmsg=f'No chunk found!',
|
||||
return get_result(retmsg=f'No chunk found!',
|
||||
retcode=RetCode.DATA_ERROR)
|
||||
return server_error_response(e)
|
||||
|
||||
|
||||
@manager.route('/chunk/create', methods=['POST'])
|
||||
@manager.route('/dataset/{dataset_id}/document/{document_id}/chunk', methods=['POST'])
|
||||
@token_required
|
||||
@validate_request("document_id", "content")
|
||||
def create(tenant_id):
|
||||
def create(tenant_id,dataset_id,document_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
doc = DocumentService.query(id=document_id, kb_id=dataset_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f"You don't own the document {document_id}.")
|
||||
req = request.json
|
||||
if not req.get("content"):
|
||||
return get_error_data_result(retmsg="`content` is required")
|
||||
md5 = hashlib.md5()
|
||||
md5.update((req["content"] + req["document_id"]).encode("utf-8"))
|
||||
md5.update((req["content"] + document_id).encode("utf-8"))
|
||||
|
||||
chunk_id = md5.hexdigest()
|
||||
d = {"id": chunk_id, "content_ltks": rag_tokenizer.tokenize(req["content"]),
|
||||
@ -530,80 +383,77 @@ def create(tenant_id):
|
||||
d["important_tks"] = rag_tokenizer.tokenize(" ".join(req.get("important_kwd", [])))
|
||||
d["create_time"] = str(datetime.datetime.now()).replace("T", " ")[:19]
|
||||
d["create_timestamp_flt"] = datetime.datetime.now().timestamp()
|
||||
d["kb_id"] = [doc.kb_id]
|
||||
d["docnm_kwd"] = doc.name
|
||||
d["doc_id"] = doc.id
|
||||
embd_id = DocumentService.get_embd_id(document_id)
|
||||
embd_mdl = TenantLLMService.model_instance(
|
||||
tenant_id, LLMType.EMBEDDING.value, embd_id)
|
||||
|
||||
try:
|
||||
e, doc = DocumentService.get_by_id(req["document_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
d["kb_id"] = [doc.kb_id]
|
||||
d["docnm_kwd"] = doc.name
|
||||
d["doc_id"] = doc.id
|
||||
v, c = embd_mdl.encode([doc.name, req["content"]])
|
||||
v = 0.1 * v[0] + 0.9 * v[1]
|
||||
d["q_%d_vec" % len(v)] = v.tolist()
|
||||
ELASTICSEARCH.upsert([d], search.index_name(tenant_id))
|
||||
|
||||
tenant_id = DocumentService.get_tenant_id(req["document_id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
DocumentService.increment_chunk_num(
|
||||
doc.id, doc.kb_id, c, 1, 0)
|
||||
d["chunk_id"] = chunk_id
|
||||
# rename keys
|
||||
key_mapping = {
|
||||
"chunk_id": "id",
|
||||
"content_with_weight": "content",
|
||||
"doc_id": "document_id",
|
||||
"important_kwd": "important_keywords",
|
||||
"kb_id": "dataset_id",
|
||||
"create_timestamp_flt": "create_timestamp",
|
||||
"create_time": "create_time",
|
||||
"document_keyword": "document",
|
||||
}
|
||||
renamed_chunk = {}
|
||||
for key, value in d.items():
|
||||
if key in key_mapping:
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_chunk[new_key] = value
|
||||
return get_result(data={"chunk": renamed_chunk})
|
||||
# return get_result(data={"chunk_id": chunk_id})
|
||||
|
||||
embd_id = DocumentService.get_embd_id(req["document_id"])
|
||||
embd_mdl = TenantLLMService.model_instance(
|
||||
tenant_id, LLMType.EMBEDDING.value, embd_id)
|
||||
|
||||
v, c = embd_mdl.encode([doc.name, req["content"]])
|
||||
v = 0.1 * v[0] + 0.9 * v[1]
|
||||
d["q_%d_vec" % len(v)] = v.tolist()
|
||||
ELASTICSEARCH.upsert([d], search.index_name(tenant_id))
|
||||
|
||||
DocumentService.increment_chunk_num(
|
||||
doc.id, doc.kb_id, c, 1, 0)
|
||||
d["chunk_id"] = chunk_id
|
||||
#rename keys
|
||||
key_mapping = {
|
||||
"chunk_id": "id",
|
||||
"content_with_weight": "content",
|
||||
"doc_id": "document_id",
|
||||
"important_kwd": "important_keywords",
|
||||
"kb_id":"dataset_id",
|
||||
"create_timestamp_flt":"create_timestamp",
|
||||
"create_time": "create_time",
|
||||
"document_keyword":"document",
|
||||
}
|
||||
renamed_chunk = {}
|
||||
for key, value in d.items():
|
||||
if key in key_mapping:
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_chunk[new_key] = value
|
||||
|
||||
return get_json_result(data={"chunk": renamed_chunk})
|
||||
# return get_json_result(data={"chunk_id": chunk_id})
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
|
||||
@manager.route('/chunk/rm', methods=['POST'])
|
||||
@manager.route('dataset/{dataset_id}/document/{document_id}/chunk', methods=['DELETE'])
|
||||
@token_required
|
||||
@validate_request("chunk_ids", "document_id")
|
||||
def rm_chunk(tenant_id):
|
||||
def rm_chunk(tenant_id,dataset_id,document_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
doc = DocumentService.query(id=document_id, kb_id=dataset_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f"You don't own the document {document_id}.")
|
||||
req = request.json
|
||||
try:
|
||||
if not ELASTICSEARCH.deleteByQuery(
|
||||
Q("ids", values=req["chunk_ids"]), search.index_name(tenant_id)):
|
||||
return get_data_error_result(retmsg="Index updating failure")
|
||||
e, doc = DocumentService.get_by_id(req["document_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
deleted_chunk_ids = req["chunk_ids"]
|
||||
chunk_number = len(deleted_chunk_ids)
|
||||
DocumentService.decrement_chunk_num(doc.id, doc.kb_id, 1, chunk_number, 0)
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
if not req.get("chunk_ids"):
|
||||
return get_error_data_result("`chunk_ids` is required")
|
||||
if not ELASTICSEARCH.deleteByQuery(
|
||||
Q("ids", values=req["chunk_ids"]), search.index_name(tenant_id)):
|
||||
return get_error_data_result(retmsg="Index updating failure")
|
||||
deleted_chunk_ids = req["chunk_ids"]
|
||||
chunk_number = len(deleted_chunk_ids)
|
||||
DocumentService.decrement_chunk_num(doc.id, doc.kb_id, 1, chunk_number, 0)
|
||||
return get_result()
|
||||
|
||||
@manager.route('/chunk/set', methods=['POST'])
|
||||
|
||||
|
||||
@manager.route('/dataset/{dataset_id}/document/{document_id}/chunk/{chunk_id}', methods=['PUT'])
|
||||
@token_required
|
||||
@validate_request("document_id", "chunk_id", "content",
|
||||
"important_keywords")
|
||||
def set(tenant_id):
|
||||
def set(tenant_id,dataset_id,document_id,chunk_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
doc = DocumentService.query(id=document_id, kb_id=dataset_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f"You don't own the document {document_id}.")
|
||||
req = request.json
|
||||
if not req.get("content"):
|
||||
return get_error_data_result("`content` is required")
|
||||
if not req.get("important_keywords"):
|
||||
return get_error_data_result("`important_keywords` is required")
|
||||
d = {
|
||||
"id": req["chunk_id"],
|
||||
"id": chunk_id,
|
||||
"content_with_weight": req["content"]}
|
||||
d["content_ltks"] = rag_tokenizer.tokenize(req["content"])
|
||||
d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
|
||||
@ -611,71 +461,54 @@ def set(tenant_id):
|
||||
d["important_tks"] = rag_tokenizer.tokenize(" ".join(req["important_keywords"]))
|
||||
if "available" in req:
|
||||
d["available_int"] = req["available"]
|
||||
embd_id = DocumentService.get_embd_id(document_id)
|
||||
embd_mdl = TenantLLMService.model_instance(
|
||||
tenant_id, LLMType.EMBEDDING.value, embd_id)
|
||||
if doc.parser_id == ParserType.QA:
|
||||
arr = [
|
||||
t for t in re.split(
|
||||
r"[\n\t]",
|
||||
req["content"]) if len(t) > 1]
|
||||
if len(arr) != 2:
|
||||
return get_error_data_result(
|
||||
retmsg="Q&A must be separated by TAB/ENTER key.")
|
||||
q, a = rmPrefix(arr[0]), rmPrefix(arr[1])
|
||||
d = beAdoc(d, arr[0], arr[1], not any(
|
||||
[rag_tokenizer.is_chinese(t) for t in q + a]))
|
||||
|
||||
try:
|
||||
tenant_id = DocumentService.get_tenant_id(req["document_id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
v, c = embd_mdl.encode([doc.name, req["content"]])
|
||||
v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
|
||||
d["q_%d_vec" % len(v)] = v.tolist()
|
||||
ELASTICSEARCH.upsert([d], search.index_name(tenant_id))
|
||||
return get_result()
|
||||
|
||||
embd_id = DocumentService.get_embd_id(req["document_id"])
|
||||
embd_mdl = TenantLLMService.model_instance(
|
||||
tenant_id, LLMType.EMBEDDING.value, embd_id)
|
||||
|
||||
e, doc = DocumentService.get_by_id(req["document_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
|
||||
if doc.parser_id == ParserType.QA:
|
||||
arr = [
|
||||
t for t in re.split(
|
||||
r"[\n\t]",
|
||||
req["content"]) if len(t) > 1]
|
||||
if len(arr) != 2:
|
||||
return get_data_error_result(
|
||||
retmsg="Q&A must be separated by TAB/ENTER key.")
|
||||
q, a = rmPrefix(arr[0]), rmPrefix(arr[1])
|
||||
d = beAdoc(d, arr[0], arr[1], not any(
|
||||
[rag_tokenizer.is_chinese(t) for t in q + a]))
|
||||
|
||||
v, c = embd_mdl.encode([doc.name, req["content"]])
|
||||
v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
|
||||
d["q_%d_vec" % len(v)] = v.tolist()
|
||||
ELASTICSEARCH.upsert([d], search.index_name(tenant_id))
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
|
||||
@manager.route('/retrieval_test', methods=['POST'])
|
||||
@manager.route('/retrieval', methods=['GET'])
|
||||
@token_required
|
||||
@validate_request("knowledgebase_id", "question")
|
||||
def retrieval_test(tenant_id):
|
||||
req = request.json
|
||||
page = int(req.get("page", 1))
|
||||
size = int(req.get("size", 30))
|
||||
req = request.args
|
||||
if not req.get("datasets"):
|
||||
return get_error_data_result("`datasets` is required.")
|
||||
for id in req.get("datasets"):
|
||||
if not KnowledgebaseService.query(id=id,tenant_id=tenant_id):
|
||||
return get_error_data_result(f"You don't own the dataset {id}.")
|
||||
if not req.get("question"):
|
||||
return get_error_data_result("`question` is required.")
|
||||
page = int(req.get("offset", 1))
|
||||
size = int(req.get("limit", 30))
|
||||
question = req["question"]
|
||||
kb_id = req["knowledgebase_id"]
|
||||
kb_id = req["datasets"]
|
||||
if isinstance(kb_id, str): kb_id = [kb_id]
|
||||
doc_ids = req.get("doc_ids", [])
|
||||
doc_ids = req.get("documents", [])
|
||||
similarity_threshold = float(req.get("similarity_threshold", 0.2))
|
||||
vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
|
||||
top = int(req.get("top_k", 1024))
|
||||
|
||||
try:
|
||||
tenants = UserTenantService.query(user_id=tenant_id)
|
||||
for kid in kb_id:
|
||||
for tenant in tenants:
|
||||
if KnowledgebaseService.query(
|
||||
tenant_id=tenant.tenant_id, id=kid):
|
||||
break
|
||||
else:
|
||||
return get_json_result(
|
||||
data=False, retmsg=f'Only owner of knowledgebase authorized for this operation.',
|
||||
retcode=RetCode.OPERATING_ERROR)
|
||||
|
||||
e, kb = KnowledgebaseService.get_by_id(kb_id[0])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Knowledgebase not found!")
|
||||
|
||||
return get_error_data_result(retmsg="Knowledgebase not found!")
|
||||
embd_mdl = TenantLLMService.model_instance(
|
||||
kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
|
||||
|
||||
@ -697,24 +530,24 @@ def retrieval_test(tenant_id):
|
||||
del c["vector"]
|
||||
|
||||
##rename keys
|
||||
renamed_chunks=[]
|
||||
renamed_chunks = []
|
||||
for chunk in ranks["chunks"]:
|
||||
key_mapping = {
|
||||
"chunk_id": "id",
|
||||
"content_with_weight": "content",
|
||||
"doc_id": "document_id",
|
||||
"important_kwd": "important_keywords",
|
||||
"docnm_kwd":"document_keyword"
|
||||
"docnm_kwd": "document_keyword"
|
||||
}
|
||||
rename_chunk={}
|
||||
rename_chunk = {}
|
||||
for key, value in chunk.items():
|
||||
new_key = key_mapping.get(key, key)
|
||||
rename_chunk[new_key] = value
|
||||
renamed_chunks.append(rename_chunk)
|
||||
ranks["chunks"] = renamed_chunks
|
||||
return get_json_result(data=ranks)
|
||||
return get_result(data=ranks)
|
||||
except Exception as e:
|
||||
if str(e).find("not_found") > 0:
|
||||
return get_json_result(data=False, retmsg=f'No chunk found! Check the chunk status please!',
|
||||
return get_result(retmsg=f'No chunk found! Check the chunk status please!',
|
||||
retcode=RetCode.DATA_ERROR)
|
||||
return server_error_response(e)
|
||||
@ -20,47 +20,18 @@ from flask import request, Response
|
||||
|
||||
from api.db import StatusEnum
|
||||
from api.db.services.dialog_service import DialogService, ConversationService, chat
|
||||
from api.settings import RetCode
|
||||
from api.utils import get_uuid
|
||||
from api.utils.api_utils import get_data_error_result
|
||||
from api.utils.api_utils import get_json_result, token_required
|
||||
from api.utils.api_utils import get_error_data_result
|
||||
from api.utils.api_utils import get_result, token_required
|
||||
|
||||
|
||||
@manager.route('/save', methods=['POST'])
|
||||
@manager.route('/chat/<chat_id>/session', methods=['POST'])
|
||||
@token_required
|
||||
def set_conversation(tenant_id):
|
||||
def create(tenant_id,chat_id):
|
||||
req = request.json
|
||||
conv_id = req.get("id")
|
||||
if "assistant_id" in req:
|
||||
req["dialog_id"] = req.pop("assistant_id")
|
||||
if "id" in req:
|
||||
del req["id"]
|
||||
conv = ConversationService.query(id=conv_id)
|
||||
if not conv:
|
||||
return get_data_error_result(retmsg="Session does not exist")
|
||||
if not DialogService.query(id=conv[0].dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_data_error_result(retmsg="You do not own the session")
|
||||
if req.get("dialog_id"):
|
||||
dia = DialogService.query(tenant_id=tenant_id, id=req["dialog_id"], status=StatusEnum.VALID.value)
|
||||
if not dia:
|
||||
return get_data_error_result(retmsg="You do not own the assistant")
|
||||
if "dialog_id" in req and not req.get("dialog_id"):
|
||||
return get_data_error_result(retmsg="assistant_id can not be empty.")
|
||||
if "message" in req:
|
||||
return get_data_error_result(retmsg="message can not be change")
|
||||
if "reference" in req:
|
||||
return get_data_error_result(retmsg="reference can not be change")
|
||||
if "name" in req and not req.get("name"):
|
||||
return get_data_error_result(retmsg="name can not be empty.")
|
||||
if not ConversationService.update_by_id(conv_id, req):
|
||||
return get_data_error_result(retmsg="Session updates error")
|
||||
return get_json_result(data=True)
|
||||
|
||||
if not req.get("dialog_id"):
|
||||
return get_data_error_result(retmsg="assistant_id is required.")
|
||||
req["dialog_id"] = chat_id
|
||||
dia = DialogService.query(tenant_id=tenant_id, id=req["dialog_id"], status=StatusEnum.VALID.value)
|
||||
if not dia:
|
||||
return get_data_error_result(retmsg="You do not own the assistant")
|
||||
return get_error_data_result(retmsg="You do not own the assistant")
|
||||
conv = {
|
||||
"id": get_uuid(),
|
||||
"dialog_id": req["dialog_id"],
|
||||
@ -68,33 +39,58 @@ def set_conversation(tenant_id):
|
||||
"message": [{"role": "assistant", "content": "Hi! I am your assistant，can I help you?"}]
|
||||
}
|
||||
if not conv.get("name"):
|
||||
return get_data_error_result(retmsg="name can not be empty.")
|
||||
return get_error_data_result(retmsg="Name can not be empty.")
|
||||
ConversationService.save(**conv)
|
||||
e, conv = ConversationService.get_by_id(conv["id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Fail to new session!")
|
||||
return get_error_data_result(retmsg="Fail to create a session!")
|
||||
conv = conv.to_dict()
|
||||
conv['messages'] = conv.pop("message")
|
||||
conv["assistant_id"] = conv.pop("dialog_id")
|
||||
conv["chat_id"] = conv.pop("dialog_id")
|
||||
del conv["reference"]
|
||||
return get_json_result(data=conv)
|
||||
return get_result(data=conv)
|
||||
|
||||
|
||||
@manager.route('/completion', methods=['POST'])
|
||||
@manager.route('/chat/<chat_id>/session/<session_id>', methods=['PUT'])
|
||||
@token_required
|
||||
def completion(tenant_id):
|
||||
def update(tenant_id,chat_id,session_id):
|
||||
req = request.json
|
||||
if "dialog_id" in req and req.get("dialog_id") != chat_id:
|
||||
return get_error_data_result(retmsg="Can't change chat_id")
|
||||
if "chat_id" in req and req.get("chat_id") != chat_id:
|
||||
return get_error_data_result(retmsg="Can't change chat_id")
|
||||
req["dialog_id"] = chat_id
|
||||
conv_id = session_id
|
||||
conv = ConversationService.query(id=conv_id,dialog_id=chat_id)
|
||||
if not conv:
|
||||
return get_error_data_result(retmsg="Session does not exist")
|
||||
if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_error_data_result(retmsg="You do not own the session")
|
||||
if "message" in req or "messages" in req:
|
||||
return get_error_data_result(retmsg="Message can not be change")
|
||||
if "reference" in req:
|
||||
return get_error_data_result(retmsg="Reference can not be change")
|
||||
if "name" in req and not req.get("name"):
|
||||
return get_error_data_result(retmsg="Name can not be empty.")
|
||||
if not ConversationService.update_by_id(conv_id, req):
|
||||
return get_error_data_result(retmsg="Session updates error")
|
||||
return get_result()
|
||||
|
||||
|
||||
@manager.route('/chat/<chat_id>/session/<session_id>/completion', methods=['POST'])
|
||||
@token_required
|
||||
def completion(tenant_id,chat_id,session_id):
|
||||
req = request.json
|
||||
# req = {"conversation_id": "9aaaca4c11d311efa461fa163e197198", "messages": [
|
||||
# {"role": "user", "content": "上海有吗？"}
|
||||
# ]}
|
||||
if "session_id" not in req:
|
||||
return get_data_error_result(retmsg="session_id is required")
|
||||
conv = ConversationService.query(id=req["session_id"])
|
||||
if not req.get("question"):
|
||||
return get_error_data_result(retmsg="Please input your question.")
|
||||
conv = ConversationService.query(id=session_id,dialog_id=chat_id)
|
||||
if not conv:
|
||||
return get_data_error_result(retmsg="Session does not exist")
|
||||
return get_error_data_result(retmsg="Session does not exist")
|
||||
conv = conv[0]
|
||||
if not DialogService.query(id=conv.dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_data_error_result(retmsg="You do not own the session")
|
||||
if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_error_data_result(retmsg="You do not own the session")
|
||||
msg = []
|
||||
question = {
|
||||
"content": req.get("question"),
|
||||
@ -108,7 +104,6 @@ def completion(tenant_id):
|
||||
msg.append(m)
|
||||
message_id = msg[-1].get("id")
|
||||
e, dia = DialogService.get_by_id(conv.dialog_id)
|
||||
del req["session_id"]
|
||||
|
||||
if not conv.reference:
|
||||
conv.reference = []
|
||||
@ -130,13 +125,13 @@ def completion(tenant_id):
|
||||
try:
|
||||
for ans in chat(dia, msg, **req):
|
||||
fillin_conv(ans)
|
||||
yield "data:" + json.dumps({"retcode": 0, "retmsg": "", "data": ans}, ensure_ascii=False) + "\n\n"
|
||||
yield "data:" + json.dumps({"code": 0, "data": ans}, ensure_ascii=False) + "\n\n"
|
||||
ConversationService.update_by_id(conv.id, conv.to_dict())
|
||||
except Exception as e:
|
||||
yield "data:" + json.dumps({"retcode": 500, "retmsg": str(e),
|
||||
yield "data:" + json.dumps({"code": 500, "message": str(e),
|
||||
"data": {"answer": "**ERROR**: " + str(e), "reference": []}},
|
||||
ensure_ascii=False) + "\n\n"
|
||||
yield "data:" + json.dumps({"retcode": 0, "retmsg": "", "data": True}, ensure_ascii=False) + "\n\n"
|
||||
yield "data:" + json.dumps({"code": 0, "data": True}, ensure_ascii=False) + "\n\n"
|
||||
|
||||
if req.get("stream", True):
|
||||
resp = Response(stream(), mimetype="text/event-stream")
|
||||
@ -153,73 +148,31 @@ def completion(tenant_id):
|
||||
fillin_conv(ans)
|
||||
ConversationService.update_by_id(conv.id, conv.to_dict())
|
||||
break
|
||||
return get_json_result(data=answer)
|
||||
return get_result(data=answer)
|
||||
|
||||
|
||||
@manager.route('/get', methods=['GET'])
|
||||
@manager.route('/chat/<chat_id>/session', methods=['GET'])
|
||||
@token_required
|
||||
def get(tenant_id):
|
||||
req = request.args
|
||||
if "id" not in req:
|
||||
return get_data_error_result(retmsg="id is required")
|
||||
conv_id = req["id"]
|
||||
conv = ConversationService.query(id=conv_id)
|
||||
if not conv:
|
||||
return get_data_error_result(retmsg="Session does not exist")
|
||||
if not DialogService.query(id=conv[0].dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_data_error_result(retmsg="You do not own the session")
|
||||
if "assistant_id" in req:
|
||||
if req["assistant_id"] != conv[0].dialog_id:
|
||||
return get_data_error_result(retmsg="The session doesn't belong to the assistant")
|
||||
conv = conv[0].to_dict()
|
||||
conv['messages'] = conv.pop("message")
|
||||
conv["assistant_id"] = conv.pop("dialog_id")
|
||||
if conv["reference"]:
|
||||
messages = conv["messages"]
|
||||
message_num = 0
|
||||
chunk_num = 0
|
||||
while message_num < len(messages):
|
||||
if message_num != 0 and messages[message_num]["role"] != "user":
|
||||
chunk_list = []
|
||||
if "chunks" in conv["reference"][chunk_num]:
|
||||
chunks = conv["reference"][chunk_num]["chunks"]
|
||||
for chunk in chunks:
|
||||
new_chunk = {
|
||||
"id": chunk["chunk_id"],
|
||||
"content": chunk["content_with_weight"],
|
||||
"document_id": chunk["doc_id"],
|
||||
"document_name": chunk["docnm_kwd"],
|
||||
"knowledgebase_id": chunk["kb_id"],
|
||||
"image_id": chunk["img_id"],
|
||||
"similarity": chunk["similarity"],
|
||||
"vector_similarity": chunk["vector_similarity"],
|
||||
"term_similarity": chunk["term_similarity"],
|
||||
"positions": chunk["positions"],
|
||||
}
|
||||
chunk_list.append(new_chunk)
|
||||
chunk_num += 1
|
||||
messages[message_num]["reference"] = chunk_list
|
||||
message_num += 1
|
||||
del conv["reference"]
|
||||
return get_json_result(data=conv)
|
||||
|
||||
|
||||
@manager.route('/list', methods=["GET"])
|
||||
@token_required
|
||||
def list(tenant_id):
|
||||
assistant_id = request.args["assistant_id"]
|
||||
if not DialogService.query(tenant_id=tenant_id, id=assistant_id, status=StatusEnum.VALID.value):
|
||||
return get_json_result(
|
||||
data=False, retmsg=f"You don't own the assistant.",
|
||||
retcode=RetCode.OPERATING_ERROR)
|
||||
convs = ConversationService.query(
|
||||
dialog_id=assistant_id,
|
||||
order_by=ConversationService.model.create_time,
|
||||
reverse=True)
|
||||
convs = [d.to_dict() for d in convs]
|
||||
def list(chat_id,tenant_id):
|
||||
if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
|
||||
return get_error_data_result(retmsg=f"You don't own the assistant {chat_id}.")
|
||||
id = request.args.get("id")
|
||||
name = request.args.get("name")
|
||||
session = ConversationService.query(id=id,name=name,dialog_id=chat_id)
|
||||
if not session:
|
||||
return get_error_data_result(retmsg="The session doesn't exist")
|
||||
page_number = int(request.args.get("page", 1))
|
||||
items_per_page = int(request.args.get("page_size", 1024))
|
||||
orderby = request.args.get("orderby", "create_time")
|
||||
if request.args.get("desc") == "False":
|
||||
desc = False
|
||||
else:
|
||||
desc = True
|
||||
convs = ConversationService.get_list(chat_id,page_number,items_per_page,orderby,desc,id,name)
|
||||
if not convs:
|
||||
return get_result(data=[])
|
||||
for conv in convs:
|
||||
conv['messages'] = conv.pop("message")
|
||||
conv["assistant_id"] = conv.pop("dialog_id")
|
||||
conv["chat"] = conv.pop("dialog_id")
|
||||
if conv["reference"]:
|
||||
messages = conv["messages"]
|
||||
message_num = 0
|
||||
@ -247,20 +200,19 @@ def list(tenant_id):
|
||||
messages[message_num]["reference"] = chunk_list
|
||||
message_num += 1
|
||||
del conv["reference"]
|
||||
return get_json_result(data=convs)
|
||||
return get_result(data=convs)
|
||||
|
||||
|
||||
@manager.route('/delete', methods=["DELETE"])
|
||||
@manager.route('/chat/<chat_id>/session', methods=["DELETE"])
|
||||
@token_required
|
||||
def delete(tenant_id):
|
||||
id = request.args.get("id")
|
||||
if not id:
|
||||
return get_data_error_result(retmsg="`id` is required in deleting operation")
|
||||
conv = ConversationService.query(id=id)
|
||||
if not conv:
|
||||
return get_data_error_result(retmsg="Session doesn't exist")
|
||||
conv = conv[0]
|
||||
if not DialogService.query(id=conv.dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_data_error_result(retmsg="You don't own the session")
|
||||
ConversationService.delete_by_id(id)
|
||||
return get_json_result(data=True)
|
||||
def delete(tenant_id,chat_id):
|
||||
if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_error_data_result(retmsg="You don't own the chat")
|
||||
ids = request.json.get("ids")
|
||||
if not ids:
|
||||
return get_error_data_result(retmsg="`ids` is required in deleting operation")
|
||||
for id in ids:
|
||||
conv = ConversationService.query(id=id,dialog_id=chat_id)
|
||||
if not conv:
|
||||
return get_error_data_result(retmsg="The chat doesn't own the session")
|
||||
ConversationService.delete_by_id(id)
|
||||
return get_result()
|
||||
|
||||
Reference in New Issue
Block a user