Replaced md5 with xxhash64 for chunk id (#4009)

### What problem does this PR solve?

Replaced md5 with xxhash64 for chunk id

### Type of change

- [x] Refactoring
This commit is contained in:
Zhichang Yu
2024-12-12 17:47:39 +08:00
committed by GitHub
parent 301f95837c
commit c8b1a564aa
5 changed files with 17 additions and 34 deletions

View File

@ -22,7 +22,7 @@ from rag.nlp import rag_tokenizer
from api.db import LLMType, ParserType
from api.db.services.llm_service import TenantLLMService
from api import settings
import hashlib
import xxhash
import re
from api.utils.api_utils import token_required
from api.db.db_models import Task
@ -984,10 +984,7 @@ def add_chunk(tenant_id, dataset_id, document_id):
return get_error_data_result(
"`questions` is required to be a list"
)
md5 = hashlib.md5()
md5.update((req["content"] + document_id).encode("utf-8"))
chunk_id = md5.hexdigest()
chunk_id = xxhash.xxh64((req["content"] + document_id).encode("utf-8")).hexdigest()
d = {
"id": chunk_id,
"content_ltks": rag_tokenizer.tokenize(req["content"]),