diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py index 23a0dc214..f5799a964 100644 --- a/rag/svr/task_executor.py +++ b/rag/svr/task_executor.py @@ -284,7 +284,7 @@ async def build_chunks(task, progress_callback): try: d = copy.deepcopy(document) d.update(chunk) - d["id"] = xxhash.xxh64((chunk["content_with_weight"] + str(d["doc_id"])).encode("utf-8")).hexdigest() + d["id"] = xxhash.xxh64((chunk["content_with_weight"] + str(d["doc_id"])).encode("utf-8", "surrogatepass")).hexdigest() d["create_time"] = str(datetime.now()).replace("T", " ")[:19] d["create_timestamp_flt"] = datetime.now().timestamp() if not d.get("image"):