pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-24 15:36:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Change knowledge base to dataset (#11976)

Browse Source 
### What problem does this PR solve?

As title

### Type of change

- [x] Refactoring

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2025-12-17 10:03:33 +08:00
committed by GitHub
parent 4d46726eb7
commit 30019dab9f
35 changed files with 92 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/apps/chunk_app.py
View File

@ -348,7 +348,7 @@ async def retrieval_test():
break
else:
return get_json_result(
data=False, message='Only owner of knowledgebase authorized for this operation.',
data=False, message='Only owner of dataset authorized for this operation.',
code=RetCode.OPERATING_ERROR)
e, kb = KnowledgebaseService.get_by_id(kb_ids[0])

2
api/apps/dialog_app.py
View File

@ -65,7 +65,7 @@ async def set_dialog():
if not is_create:
if not req.get("kb_ids", []) and not prompt_config.get("tavily_api_key") and "{knowledge}" in prompt_config['system']:
return get_data_error_result(message="Please remove `{knowledge}` in system prompt since no knowledge base / Tavily used here.")
return get_data_error_result(message="Please remove `{knowledge}` in system prompt since no dataset / Tavily used here.")
for p in prompt_config["parameters"]:
if p["optional"]:

20
api/apps/document_app.py
View File

@ -70,7 +70,7 @@ async def upload():
e, kb = KnowledgebaseService.get_by_id(kb_id)
if not e:
raise LookupError("Can't find this knowledgebase!")
raise LookupError("Can't find this dataset!")
if not check_kb_team_permission(kb, current_user.id):
return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
@ -99,7 +99,7 @@ async def web_crawl():
return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
e, kb = KnowledgebaseService.get_by_id(kb_id)
if not e:
raise LookupError("Can't find this knowledgebase!")
raise LookupError("Can't find this dataset!")
if check_kb_team_permission(kb, current_user.id):
return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
@ -169,10 +169,10 @@ async def create():
try:
e, kb = KnowledgebaseService.get_by_id(kb_id)
if not e:
return get_data_error_result(message="Can't find this knowledgebase!")
return get_data_error_result(message="Can't find this dataset!")
if DocumentService.query(name=req["name"], kb_id=kb_id):
return get_data_error_result(message="Duplicated document name in the same knowledgebase.")
return get_data_error_result(message="Duplicated document name in the same dataset.")
kb_root_folder = FileService.get_kb_folder(kb.tenant_id)
if not kb_root_folder:
@ -219,7 +219,7 @@ async def list_docs():
if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
break
else:
return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
return get_json_result(data=False, message="Only owner of dataset authorized for this operation.", code=RetCode.OPERATING_ERROR)
keywords = request.args.get("keywords", "")
page_number = int(request.args.get("page", 0))
@ -293,7 +293,7 @@ async def get_filter():
if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
break
else:
return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
return get_json_result(data=False, message="Only owner of dataset authorized for this operation.", code=RetCode.OPERATING_ERROR)
keywords = req.get("keywords", "")
@ -343,7 +343,7 @@ async def metadata_summary():
if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
break
else:
return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
return get_json_result(data=False, message="Only owner of dataset authorized for this operation.", code=RetCode.OPERATING_ERROR)
try:
summary = DocumentService.get_metadata_summary(kb_id)
@ -365,7 +365,7 @@ async def metadata_update():
if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
break
else:
return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
return get_json_result(data=False, message="Only owner of dataset authorized for this operation.", code=RetCode.OPERATING_ERROR)
selector = req.get("selector", {}) or {}
updates = req.get("updates", []) or []
@ -454,7 +454,7 @@ async def change_status():
continue
e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
if not e:
result[doc_id] = {"error": "Can't find this knowledgebase!"}
result[doc_id] = {"error": "Can't find this dataset!"}
continue
if not DocumentService.update_by_id(doc_id, {"status": str(status)}):
result[doc_id] = {"error": "Database error (Document update)!"}
@ -562,7 +562,7 @@ async def rename():
for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
if d.name == req["name"]:
return get_data_error_result(message="Duplicated document name in the same knowledgebase.")
return get_data_error_result(message="Duplicated document name in the same dataset.")
if not DocumentService.update_by_id(req["doc_id"], {"name": req["name"]}):
return get_data_error_result(message="Database error (Document rename)!")

2
api/apps/file2document_app.py
View File

@ -68,7 +68,7 @@ async def convert():
e, kb = KnowledgebaseService.get_by_id(kb_id)
if not e:
return get_data_error_result(
message="Can't find this knowledgebase!")
message="Can't find this dataset!")
e, file = FileService.get_by_id(id)
if not e:
return get_data_error_result(

12
api/apps/kb_app.py
View File

@ -93,19 +93,19 @@ async def update():
if not KnowledgebaseService.query(
created_by=current_user.id, id=req["kb_id"]):
return get_json_result(
data=False, message='Only owner of knowledgebase authorized for this operation.',
data=False, message='Only owner of dataset authorized for this operation.',
code=RetCode.OPERATING_ERROR)
e, kb = KnowledgebaseService.get_by_id(req["kb_id"])
if not e:
return get_data_error_result(
message="Can't find this knowledgebase!")
message="Can't find this dataset!")
if req["name"].lower() != kb.name.lower() \
and len(
KnowledgebaseService.query(name=req["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) >= 1:
return get_data_error_result(
message="Duplicated knowledgebase name.")
message="Duplicated dataset name.")
del req["kb_id"]
connectors = []
@ -162,12 +162,12 @@ def detail():
break
else:
return get_json_result(
data=False, message='Only owner of knowledgebase authorized for this operation.',
data=False, message='Only owner of dataset authorized for this operation.',
code=RetCode.OPERATING_ERROR)
kb = KnowledgebaseService.get_detail(kb_id)
if not kb:
return get_data_error_result(
message="Can't find this knowledgebase!")
message="Can't find this dataset!")
kb["size"] = DocumentService.get_total_size_by_kb_id(kb_id=kb["id"],keywords="", run_status=[], types=[])
kb["connectors"] = Connector2KbService.list_connectors(kb_id)
@ -232,7 +232,7 @@ async def rm():
created_by=current_user.id, id=req["kb_id"])
if not kbs:
return get_json_result(
data=False, message='Only owner of knowledgebase authorized for this operation.',
data=False, message='Only owner of dataset authorized for this operation.',
code=RetCode.OPERATING_ERROR)
def _rm_sync():

2
api/apps/sdk/chat.py
View File

@ -92,7 +92,7 @@ async def create(tenant_id):
req["tenant_id"] = tenant_id
# prompt more parameter
default_prompt = {
"system": """You are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence "The answer you are looking for is not found in the knowledge base!" Answers need to consider chat history.
"system": """You are an intelligent assistant. Please summarize the content of the dataset to answer the question. Please list the data in the dataset and answer in detail. When all dataset content is irrelevant to the question, your answer must include the sentence "The answer you are looking for is not found in the dataset!" Answers need to consider chat history.
Here is the knowledge base:
{knowledge}
The above is the knowledge base.""",

2
api/apps/sdk/doc.py
View File

@ -237,7 +237,7 @@ async def update_doc(tenant_id, dataset_id, document_id):
return get_error_data_result(message="You don't own the dataset.")
e, kb = KnowledgebaseService.get_by_id(dataset_id)
if not e:
return get_error_data_result(message="Can't find this knowledgebase!")
return get_error_data_result(message="Can't find this dataset!")
doc = DocumentService.query(kb_id=dataset_id, id=document_id)
if not doc:
return get_error_data_result(message="The dataset doesn't own the document.")

2
api/apps/sdk/files.py
View File

@ -744,7 +744,7 @@ async def convert(tenant_id):
e, kb = KnowledgebaseService.get_by_id(kb_id)
if not e:
return get_json_result(
message="Can't find this knowledgebase!", code=RetCode.NOT_FOUND)
message="Can't find this dataset!", code=RetCode.NOT_FOUND)
e, file = FileService.get_by_id(id)
if not e:
return get_json_result(

2
api/apps/sdk/session.py
View File

@ -998,7 +998,7 @@ async def retrieval_test_embedded():
tenant_ids.append(tenant.tenant_id)
break
else:
return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.",
return get_json_result(data=False, message="Only owner of dataset authorized for this operation.",
code=RetCode.OPERATING_ERROR)
e, kb = KnowledgebaseService.get_by_id(kb_ids[0])

6
api/db/joint_services/user_account_service.py
View File

@ -153,7 +153,7 @@ def delete_user_data(user_id: str) -> dict:
done_msg += "Start to delete owned tenant.\n"
tenant_id = owned_tenant[0]["tenant_id"]
kb_ids = KnowledgebaseService.get_kb_ids(usr.id)
# step1.1 delete knowledgebase related file and info
# step1.1 delete dataset related file and info
if kb_ids:
# step1.1.1 delete files in storage, remove bucket
for kb_id in kb_ids:
@ -182,7 +182,7 @@ def delete_user_data(user_id: str) -> dict:
search.index_name(tenant_id), kb_ids)
done_msg += f"- Deleted {r} chunk records.\n"
kb_delete_res = KnowledgebaseService.delete_by_ids(kb_ids)
done_msg += f"- Deleted {kb_delete_res} knowledgebase records.\n"
done_msg += f"- Deleted {kb_delete_res} dataset records.\n"
# step1.1.4 delete agents
agent_delete_res = delete_user_agents(usr.id)
done_msg += f"- Deleted {agent_delete_res['agents_deleted_count']} agent, {agent_delete_res['version_deleted_count']} versions records.\n"
@ -258,7 +258,7 @@ def delete_user_data(user_id: str) -> dict:
# step2.1.5 delete document record
doc_delete_res = DocumentService.delete_by_ids([d['id'] for d in created_documents])
done_msg += f"- Deleted {doc_delete_res} documents.\n"
# step2.1.6 update knowledge base doc&chunk&token cnt
# step2.1.6 update dataset doc&chunk&token cnt
for kb_id, doc_num in kb_doc_info.items():
KnowledgebaseService.decrease_document_num_in_delete(kb_id, doc_num)

6
api/db/services/document_service.py
View File

@ -1082,12 +1082,12 @@ def doc_upload_and_parse(conversation_id, file_objs, user_id):
e, dia = DialogService.get_by_id(conv.dialog_id)
if not dia.kb_ids:
raise LookupError("No knowledge base associated with this conversation. "
"Please add a knowledge base before uploading documents")
raise LookupError("No dataset associated with this conversation. "
"Please add a dataset before uploading documents")
kb_id = dia.kb_ids[0]
e, kb = KnowledgebaseService.get_by_id(kb_id)
if not e:
raise LookupError("Can't find this knowledgebase!")
raise LookupError("Can't find this dataset!")
embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING, llm_name=kb.embd_id, lang=kb.language)

10
api/db/services/file_service.py
View File

@ -94,11 +94,11 @@ class FileService(CommonService):
@classmethod
@DB.connection_context()
def get_kb_id_by_file_id(cls, file_id):
# Get knowledge base IDs associated with a file
# Get dataset IDs associated with a file
# Args:
# file_id: File ID
# Returns:
# List of dictionaries containing knowledge base IDs and names
# List of dictionaries containing dataset IDs and names
kbs = (
cls.model.select(*[Knowledgebase.id, Knowledgebase.name])
.join(File2Document, on=(File2Document.file_id == file_id))
@ -247,7 +247,7 @@ class FileService(CommonService):
@classmethod
@DB.connection_context()
def get_kb_folder(cls, tenant_id):
# Get knowledge base folder for tenant
# Get dataset folder for tenant
# Args:
# tenant_id: Tenant ID
# Returns:
@ -263,7 +263,7 @@ class FileService(CommonService):
@classmethod
@DB.connection_context()
def new_a_file_from_kb(cls, tenant_id, name, parent_id, ty=FileType.FOLDER.value, size=0, location=""):
# Create a new file from knowledge base
# Create a new file from dataset
# Args:
# tenant_id: Tenant ID
# name: File name
@ -292,7 +292,7 @@ class FileService(CommonService):
@classmethod
@DB.connection_context()
def init_knowledgebase_docs(cls, root_id, tenant_id):
# Initialize knowledge base documents
# Initialize dataset documents
# Args:
# root_id: Root folder ID
# tenant_id: Tenant ID

66
api/db/services/knowledgebase_service.py
View File

@ -30,9 +30,9 @@ from api.utils.api_utils import get_parser_config, get_data_error_result
class KnowledgebaseService(CommonService):
"""Service class for managing knowledge base operations.
"""Service class for managing dataset operations.
This class extends CommonService to provide specialized functionality for knowledge base
This class extends CommonService to provide specialized functionality for dataset
management, including document parsing status tracking, access control, and configuration
management. It handles operations such as listing, creating, updating, and deleting
knowledge bases, as well as managing their associated documents and permissions.
@ -41,7 +41,7 @@ class KnowledgebaseService(CommonService):
- Document parsing status verification
- Knowledge base access control
- Parser configuration management
- Tenant-based knowledge base organization
- Tenant-based dataset organization
Attributes:
model: The Knowledgebase model class for database operations.
@ -51,18 +51,18 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def accessible4deletion(cls, kb_id, user_id):
"""Check if a knowledge base can be deleted by a specific user.
"""Check if a dataset can be deleted by a specific user.
This method verifies whether a user has permission to delete a knowledge base
by checking if they are the creator of that knowledge base.
This method verifies whether a user has permission to delete a dataset
by checking if they are the creator of that dataset.
Args:
kb_id (str): The unique identifier of the knowledge base to check.
kb_id (str): The unique identifier of the dataset to check.
user_id (str): The unique identifier of the user attempting the deletion.
Returns:
bool: True if the user has permission to delete the knowledge base,
False if the user doesn't have permission or the knowledge base doesn't exist.
bool: True if the user has permission to delete the dataset,
False if the user doesn't have permission or the dataset doesn't exist.
Example:
>>> KnowledgebaseService.accessible4deletion("kb123", "user456")
@ -71,10 +71,10 @@ class KnowledgebaseService(CommonService):
Note:
- This method only checks creator permissions
- A return value of False can mean either:
1. The knowledge base doesn't exist
2. The user is not the creator of the knowledge base
1. The dataset doesn't exist
2. The user is not the creator of the dataset
"""
# Check if a knowledge base can be deleted by a user
# Check if a dataset can be deleted by a user
docs = cls.model.select(
cls.model.id).where(cls.model.id == kb_id, cls.model.created_by == user_id).paginate(0, 1)
docs = docs.dicts()
@ -85,7 +85,7 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def is_parsed_done(cls, kb_id):
# Check if all documents in the knowledge base have completed parsing
# Check if all documents in the dataset have completed parsing
#
# Args:
# kb_id: Knowledge base ID
@ -96,13 +96,13 @@ class KnowledgebaseService(CommonService):
from common.constants import TaskStatus
from api.db.services.document_service import DocumentService
# Get knowledge base information
# Get dataset information
kbs = cls.query(id=kb_id)
if not kbs:
return False, "Knowledge base not found"
kb = kbs[0]
# Get all documents in the knowledge base
# Get all documents in the dataset
docs, _ = DocumentService.get_by_kb_id(kb_id, 1, 1000, "create_time", True, "", [], [])
# Check parsing status of each document
@ -119,9 +119,9 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def list_documents_by_ids(cls, kb_ids):
# Get document IDs associated with given knowledge base IDs
# Get document IDs associated with given dataset IDs
# Args:
# kb_ids: List of knowledge base IDs
# kb_ids: List of dataset IDs
# Returns:
# List of document IDs
doc_ids = cls.model.select(Document.id.alias("document_id")).join(Document, on=(cls.model.id == Document.kb_id)).where(
@ -235,11 +235,11 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def get_kb_ids(cls, tenant_id):
# Get all knowledge base IDs for a tenant
# Get all dataset IDs for a tenant
# Args:
# tenant_id: Tenant ID
# Returns:
# List of knowledge base IDs
# List of dataset IDs
fields = [
cls.model.id,
]
@ -250,11 +250,11 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def get_detail(cls, kb_id):
# Get detailed information about a knowledge base
# Get detailed information about a dataset
# Args:
# kb_id: Knowledge base ID
# Returns:
# Dictionary containing knowledge base details
# Dictionary containing dataset details
fields = [
cls.model.id,
cls.model.embd_id,
@ -294,13 +294,13 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def update_parser_config(cls, id, config):
# Update parser configuration for a knowledge base
# Update parser configuration for a dataset
# Args:
# id: Knowledge base ID
# config: New parser configuration
e, m = cls.get_by_id(id)
if not e:
raise LookupError(f"knowledgebase({id}) not found.")
raise LookupError(f"dataset({id}) not found.")
def dfs_update(old, new):
# Deep update of nested configuration
@ -325,7 +325,7 @@ class KnowledgebaseService(CommonService):
def delete_field_map(cls, id):
e, m = cls.get_by_id(id)
if not e:
raise LookupError(f"knowledgebase({id}) not found.")
raise LookupError(f"dataset({id}) not found.")
m.parser_config.pop("field_map", None)
cls.update_by_id(id, {"parser_config": m.parser_config})
@ -335,7 +335,7 @@ class KnowledgebaseService(CommonService):
def get_field_map(cls, ids):
# Get field mappings for knowledge bases
# Args:
# ids: List of knowledge base IDs
# ids: List of dataset IDs
# Returns:
# Dictionary of field mappings
conf = {}
@ -347,7 +347,7 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def get_by_name(cls, kb_name, tenant_id):
# Get knowledge base by name and tenant ID
# Get dataset by name and tenant ID
# Args:
# kb_name: Knowledge base name
# tenant_id: Tenant ID
@ -365,9 +365,9 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def get_all_ids(cls):
# Get all knowledge base IDs
# Get all dataset IDs
# Returns:
# List of all knowledge base IDs
# List of all dataset IDs
return [m["id"] for m in cls.model.select(cls.model.id).dicts()]
@ -471,7 +471,7 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def accessible(cls, kb_id, user_id):
# Check if a knowledge base is accessible by a user
# Check if a dataset is accessible by a user
# Args:
# kb_id: Knowledge base ID
# user_id: User ID
@ -488,12 +488,12 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def get_kb_by_id(cls, kb_id, user_id):
# Get knowledge base by ID and user ID
# Get dataset by ID and user ID
# Args:
# kb_id: Knowledge base ID
# user_id: User ID
# Returns:
# List containing knowledge base information
# List containing dataset information
kbs = cls.model.select().join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
).where(cls.model.id == kb_id, UserTenant.user_id == user_id).paginate(0, 1)
kbs = kbs.dicts()
@ -502,12 +502,12 @@ class KnowledgebaseService(CommonService):
@classmethod
@DB.connection_context()
def get_kb_by_name(cls, kb_name, user_id):
# Get knowledge base by name and user ID
# Get dataset by name and user ID
# Args:
# kb_name: Knowledge base name
# user_id: User ID
# Returns:
# List containing knowledge base information
# List containing dataset information
kbs = cls.model.select().join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
).where(cls.model.name == kb_name, UserTenant.user_id == user_id).paginate(0, 1)
kbs = kbs.dicts()

2
api/db/services/pipeline_operation_log_service.py
View File

@ -121,7 +121,7 @@ class PipelineOperationLogService(CommonService):
else:
ok, kb_info = KnowledgebaseService.get_by_id(document.kb_id)
if not ok:
raise RuntimeError(f"Cannot find knowledge base {document.kb_id} for referred_document {referred_document_id}")
raise RuntimeError(f"Cannot find dataset {document.kb_id} for referred_document {referred_document_id}")
tenant_id = kb_info.tenant_id
title = document.parser_id

2
api/db/services/task_service.py
View File

@ -76,7 +76,7 @@ class TaskService(CommonService):
"""Retrieve detailed task information by task ID.
This method fetches comprehensive task details including associated document,
knowledge base, and tenant information. It also handles task retry logic and
dataset, and tenant information. It also handles task retry logic and
progress updates.
Args: