pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2026-01-26 13:16:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: enhance metadata arranging. (#12745)

Browse Source 
### What problem does this PR solve?
#11564

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
Kevin Hu
2026-01-22 15:34:08 +08:00
committed by GitHub
parent bc7b864a6c
commit 3beb85efa0
10 changed files with 195 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
test/testcases/test_http_api/test_file_management_within_dataset/test_metadata_summary.py
View File

@ -16,8 +16,8 @@
# Although the docs group this under "chunk management," the backend aggregates
# Document.meta_fields via document_service#get_metadata_summary and the test
# uses update_document, so it belongs with file/document management tests.
import pytest
from common import metadata_summary, update_document
# import pytest
#from common import metadata_summary, update_document
def _summary_to_counts(summary):
@ -28,25 +28,29 @@ def _summary_to_counts(summary):
class TestMetadataSummary:
@pytest.mark.p2
def test_metadata_summary_counts(self, HttpApiAuth, add_documents_func):
dataset_id, document_ids = add_documents_func
payloads = [
{"tags": ["foo", "bar"], "author": "alice"},
{"tags": ["foo"], "author": "bob"},
{"tags": ["bar", "baz"], "author": None},
]
for doc_id, meta_fields in zip(document_ids, payloads):
res = update_document(HttpApiAuth, dataset_id, doc_id, {"meta_fields": meta_fields})
assert res["code"] == 0, res
pass
res = metadata_summary(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
summary = res["data"]["summary"]
counts = _summary_to_counts(summary)
assert counts["tags"]["foo"] == 2, counts
assert counts["tags"]["bar"] == 2, counts
assert counts["tags"]["baz"] == 1, counts
assert counts["author"]["alice"] == 1, counts
assert counts["author"]["bob"] == 1, counts
assert "None" not in counts["author"], counts
# Alteration of API
# TODO
#@pytest.mark.p2
#def test_metadata_summary_counts(self, HttpApiAuth, add_documents_func):
# dataset_id, document_ids = add_documents_func
# payloads = [
# {"tags": ["foo", "bar"], "author": "alice"},
# {"tags": ["foo"], "author": "bob"},
# {"tags": ["bar", "baz"], "author": None},
# ]
# for doc_id, meta_fields in zip(document_ids, payloads):
# res = update_document(HttpApiAuth, dataset_id, doc_id, {"meta_fields": meta_fields})
# assert res["code"] == 0, res
# res = metadata_summary(HttpApiAuth, dataset_id)
# assert res["code"] == 0, res
# summary = res["data"]["summary"]
# counts = _summary_to_counts(summary)
# assert counts["tags"]["foo"] == 2, counts
# assert counts["tags"]["bar"] == 2, counts
# assert counts["tags"]["baz"] == 1, counts
# assert counts["author"]["alice"] == 1, counts
# assert counts["author"]["bob"] == 1, counts
# assert "None" not in counts["author"], counts

111
test/testcases/test_web_api/test_document_app/test_document_metadata.py
View File

@ -19,7 +19,6 @@ from common import (
document_filter,
document_infos,
document_metadata_summary,
document_metadata_update,
document_rename,
document_set_meta,
document_update_metadata_setting,
@ -48,19 +47,23 @@ class TestAuthorization:
assert res["code"] == expected_code, res
assert expected_fragment in res["message"], res
@pytest.mark.p2
@pytest.mark.parametrize("invalid_auth, expected_code, expected_fragment", INVALID_AUTH_CASES)
def test_metadata_summary_auth_invalid(self, invalid_auth, expected_code, expected_fragment):
res = document_metadata_summary(invalid_auth, {"kb_id": "kb_id"})
assert res["code"] == expected_code, res
assert expected_fragment in res["message"], res
## The inputs has been changed to add 'doc_ids'
## TODO:
#@pytest.mark.p2
#@pytest.mark.parametrize("invalid_auth, expected_code, expected_fragment", INVALID_AUTH_CASES)
#def test_metadata_summary_auth_invalid(self, invalid_auth, expected_code, expected_fragment):
# res = document_metadata_summary(invalid_auth, {"kb_id": "kb_id"})
# assert res["code"] == expected_code, res
# assert expected_fragment in res["message"], res
@pytest.mark.p2
@pytest.mark.parametrize("invalid_auth, expected_code, expected_fragment", INVALID_AUTH_CASES)
def test_metadata_update_auth_invalid(self, invalid_auth, expected_code, expected_fragment):
res = document_metadata_update(invalid_auth, {"kb_id": "kb_id", "selector": {"document_ids": ["doc_id"]}, "updates": []})
assert res["code"] == expected_code, res
assert expected_fragment in res["message"], res
## The inputs has been changed to deprecate 'selector'
## TODO:
#@pytest.mark.p2
#@pytest.mark.parametrize("invalid_auth, expected_code, expected_fragment", INVALID_AUTH_CASES)
#def test_metadata_update_auth_invalid(self, invalid_auth, expected_code, expected_fragment):
# res = document_metadata_update(invalid_auth, {"kb_id": "kb_id", "selector": {"document_ids": ["doc_id"]}, "updates": []})
# assert res["code"] == expected_code, res
# assert expected_fragment in res["message"], res
@pytest.mark.p2
@pytest.mark.parametrize("invalid_auth, expected_code, expected_fragment", INVALID_AUTH_CASES)
@ -108,38 +111,44 @@ class TestDocumentMetadata:
assert len(res["data"]) == 1, res
assert res["data"][0]["id"] == doc_id, res
@pytest.mark.p2
def test_metadata_summary(self, WebApiAuth, add_document_func):
kb_id, _ = add_document_func
res = document_metadata_summary(WebApiAuth, {"kb_id": kb_id})
assert res["code"] == 0, res
assert isinstance(res["data"]["summary"], dict), res
## The inputs has been changed to add 'doc_ids'
## TODO:
#@pytest.mark.p2
#def test_metadata_summary(self, WebApiAuth, add_document_func):
# kb_id, _ = add_document_func
# res = document_metadata_summary(WebApiAuth, {"kb_id": kb_id})
# assert res["code"] == 0, res
# assert isinstance(res["data"]["summary"], dict), res
@pytest.mark.p2
def test_metadata_update(self, WebApiAuth, add_document_func):
kb_id, doc_id = add_document_func
payload = {
"kb_id": kb_id,
"selector": {"document_ids": [doc_id]},
"updates": [{"key": "author", "value": "alice"}],
"deletes": [],
}
res = document_metadata_update(WebApiAuth, payload)
assert res["code"] == 0, res
assert res["data"]["matched_docs"] == 1, res
info_res = document_infos(WebApiAuth, {"doc_ids": [doc_id]})
assert info_res["code"] == 0, info_res
meta_fields = info_res["data"][0].get("meta_fields", {})
assert meta_fields.get("author") == "alice", info_res
@pytest.mark.p2
def test_update_metadata_setting(self, WebApiAuth, add_document_func):
_, doc_id = add_document_func
metadata = {"source": "test"}
res = document_update_metadata_setting(WebApiAuth, {"doc_id": doc_id, "metadata": metadata})
assert res["code"] == 0, res
assert res["data"]["id"] == doc_id, res
assert res["data"]["parser_config"]["metadata"] == metadata, res
## The inputs has been changed to deprecate 'selector'
## TODO:
#@pytest.mark.p2
#def test_metadata_update(self, WebApiAuth, add_document_func):
# kb_id, doc_id = add_document_func
# payload = {
# "kb_id": kb_id,
# "selector": {"document_ids": [doc_id]},
# "updates": [{"key": "author", "value": "alice"}],
# "deletes": [],
# }
# res = document_metadata_update(WebApiAuth, payload)
# assert res["code"] == 0, res
# assert res["data"]["matched_docs"] == 1, res
# info_res = document_infos(WebApiAuth, {"doc_ids": [doc_id]})
# assert info_res["code"] == 0, info_res
# meta_fields = info_res["data"][0].get("meta_fields", {})
# assert meta_fields.get("author") == "alice", info_res
## The inputs has been changed to deprecate 'selector'
## TODO:
#@pytest.mark.p2
#def test_update_metadata_setting(self, WebApiAuth, add_document_func):
# _, doc_id = add_document_func
# metadata = {"source": "test"}
# res = document_update_metadata_setting(WebApiAuth, {"doc_id": doc_id, "metadata": metadata})
# assert res["code"] == 0, res
# assert res["data"]["id"] == doc_id, res
# assert res["data"]["parser_config"]["metadata"] == metadata, res
@pytest.mark.p2
def test_change_status(self, WebApiAuth, add_document_func):
@ -189,12 +198,14 @@ class TestDocumentMetadataNegative:
assert res["code"] == 101, res
assert "KB ID" in res["message"], res
@pytest.mark.p3
def test_metadata_update_missing_kb_id(self, WebApiAuth, add_document_func):
_, doc_id = add_document_func
res = document_metadata_update(WebApiAuth, {"selector": {"document_ids": [doc_id]}, "updates": []})
assert res["code"] == 101, res
assert "KB ID" in res["message"], res
## The inputs has been changed to deprecate 'selector'
## TODO:
#@pytest.mark.p3
#def test_metadata_update_missing_kb_id(self, WebApiAuth, add_document_func):
# _, doc_id = add_document_func
# res = document_metadata_update(WebApiAuth, {"selector": {"document_ids": [doc_id]}, "updates": []})
# assert res["code"] == 101, res
# assert "KB ID" in res["message"], res
@pytest.mark.p3
def test_infos_invalid_doc_id(self, WebApiAuth):