Compare commits

...

5 Commits

Author SHA1 Message Date
Ted
ad03ede7cd fix(sdk): add cancel_all_task_of call in stop_parsing endpoint (#11748)
## Problem
The SDK API endpoint `DELETE /datasets/{dataset_id}/chunks` only updates
database status but does not send cancellation signal via Redis, causing
background parsing tasks to continue and eventually complete (status
becomes DONE instead of CANCEL).

## Root Cause
The SDK endpoint was missing the `cancel_all_task_of(id)` call that the
web API
([api/apps/document_app.py](cci:7://file:///d:/workspace1/ragflow-admin/api/apps/document_app.py:0:0-0:0))
uses to properly stop background tasks.

## Solution
Added `cancel_all_task_of(id)` call in the
[stop_parsing](cci:1://file:///d:/workspace1/ragflow/api/apps/sdk/doc.py:785:0-855:23)
function to send cancellation signal via Redis, consistent with the web
API behavior.

## Related Issue
Fixes #11745

Co-authored-by: tedhappy <tedhappy@users.noreply.github.com>
2025-12-04 19:29:06 +08:00
468e4042c2 Feat: Display the ID of the code image in the dialog. #10427 (#11746)
### What problem does this PR solve?

Feat: Display the ID of the code image in the dialog.   #10427

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
2025-12-04 18:49:55 +08:00
af1344033d Delete:remove unused tests (#11749)
### What problem does this PR solve?

change:
remove unused tests
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-12-04 18:49:32 +08:00
4012d65b3c Feat: update front end for confluence connector (#11747)
### What problem does this PR solve?

Feat: update front end for confluence connector

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
2025-12-04 18:49:13 +08:00
e2bc1a3478 Feat: add more attribute for confluence connector. (#11743)
### What problem does this PR solve?

Feat: add more attribute for confluence connector. 

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
2025-12-04 17:28:03 +08:00
8 changed files with 274 additions and 913 deletions

View File

@ -33,7 +33,7 @@ from api.db.services.file_service import FileService
from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.llm_service import LLMBundle
from api.db.services.tenant_llm_service import TenantLLMService
from api.db.services.task_service import TaskService, queue_tasks
from api.db.services.task_service import TaskService, queue_tasks, cancel_all_task_of
from api.db.services.dialog_service import meta_filter, convert_conditions
from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_parser_config, get_result, server_error_response, token_required, \
get_request_json
@ -835,6 +835,8 @@ async def stop_parsing(tenant_id, dataset_id):
return get_error_data_result(message=f"You don't own the document {id}.")
if int(doc[0].progress) == 1 or doc[0].progress == 0:
return get_error_data_result("Can't stop parsing document with progress at 0 or 1")
# Send cancellation signal via Redis to stop background task
cancel_all_task_of(id)
info = {"run": "2", "progress": 0, "chunk_num": 0}
DocumentService.update_by_id(id, info)
settings.docStoreConn.delete({"doc_id": doc[0].id}, search.index_name(tenant_id), dataset_id)

View File

@ -157,11 +157,30 @@ class Confluence(SyncBase):
from common.data_source.config import DocumentSource
from common.data_source.interfaces import StaticCredentialsProvider
index_mode = (self.conf.get("index_mode") or "everything").lower()
if index_mode not in {"everything", "space", "page"}:
index_mode = "everything"
space = ""
page_id = ""
index_recursively = False
if index_mode == "space":
space = (self.conf.get("space") or "").strip()
if not space:
raise ValueError("Space Key is required when indexing a specific Confluence space.")
elif index_mode == "page":
page_id = (self.conf.get("page_id") or "").strip()
if not page_id:
raise ValueError("Page ID is required when indexing a specific Confluence page.")
index_recursively = bool(self.conf.get("index_recursively", False))
self.connector = ConfluenceConnector(
wiki_base=self.conf["wiki_base"],
space=self.conf.get("space", ""),
is_cloud=self.conf.get("is_cloud", True),
# page_id=self.conf.get("page_id", ""),
space=space,
page_id=page_id,
index_recursively=index_recursively,
)
credentials_provider = StaticCredentialsProvider(tenant_id=task["tenant_id"], connector_name=DocumentSource.CONFLUENCE, credential_json=self.conf["credentials"])

View File

@ -1,323 +0,0 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Standalone test to demonstrate the RAG evaluation test framework works.
This test doesn't require RAGFlow dependencies.
"""
import pytest
from unittest.mock import Mock
class TestEvaluationFrameworkDemo:
"""Demo tests to verify the evaluation test framework is working"""
def test_basic_assertion(self):
"""Test basic assertion works"""
assert 1 + 1 == 2
def test_mock_evaluation_service(self):
"""Test mocking evaluation service"""
mock_service = Mock()
mock_service.create_dataset.return_value = (True, "dataset_123")
success, dataset_id = mock_service.create_dataset(
name="Test Dataset",
kb_ids=["kb_1"]
)
assert success is True
assert dataset_id == "dataset_123"
mock_service.create_dataset.assert_called_once()
def test_mock_test_case_addition(self):
"""Test mocking test case addition"""
mock_service = Mock()
mock_service.add_test_case.return_value = (True, "case_123")
success, case_id = mock_service.add_test_case(
dataset_id="dataset_123",
question="Test question?",
reference_answer="Test answer"
)
assert success is True
assert case_id == "case_123"
def test_mock_evaluation_run(self):
"""Test mocking evaluation run"""
mock_service = Mock()
mock_service.start_evaluation.return_value = (True, "run_123")
success, run_id = mock_service.start_evaluation(
dataset_id="dataset_123",
dialog_id="dialog_456",
user_id="user_1"
)
assert success is True
assert run_id == "run_123"
def test_mock_metrics_computation(self):
"""Test mocking metrics computation"""
mock_service = Mock()
# Mock retrieval metrics
metrics = {
"precision": 0.85,
"recall": 0.78,
"f1_score": 0.81,
"hit_rate": 1.0,
"mrr": 0.9
}
mock_service._compute_retrieval_metrics.return_value = metrics
result = mock_service._compute_retrieval_metrics(
retrieved_ids=["chunk_1", "chunk_2", "chunk_3"],
relevant_ids=["chunk_1", "chunk_2", "chunk_4"]
)
assert result["precision"] == 0.85
assert result["recall"] == 0.78
assert result["f1_score"] == 0.81
def test_mock_recommendations(self):
"""Test mocking recommendations"""
mock_service = Mock()
recommendations = [
{
"issue": "Low Precision",
"severity": "high",
"suggestions": [
"Increase similarity_threshold",
"Enable reranking"
]
}
]
mock_service.get_recommendations.return_value = recommendations
recs = mock_service.get_recommendations("run_123")
assert len(recs) == 1
assert recs[0]["issue"] == "Low Precision"
assert len(recs[0]["suggestions"]) == 2
@pytest.mark.parametrize("precision,recall,expected_f1", [
(1.0, 1.0, 1.0),
(0.8, 0.6, 0.69),
(0.5, 0.5, 0.5),
(0.0, 0.0, 0.0),
])
def test_f1_score_calculation(self, precision, recall, expected_f1):
"""Test F1 score calculation with different inputs"""
if precision + recall > 0:
f1 = 2 * (precision * recall) / (precision + recall)
else:
f1 = 0.0
assert abs(f1 - expected_f1) < 0.01
def test_dataset_list_structure(self):
"""Test dataset list structure"""
mock_service = Mock()
expected_result = {
"total": 3,
"datasets": [
{"id": "dataset_1", "name": "Dataset 1"},
{"id": "dataset_2", "name": "Dataset 2"},
{"id": "dataset_3", "name": "Dataset 3"}
]
}
mock_service.list_datasets.return_value = expected_result
result = mock_service.list_datasets(
tenant_id="tenant_1",
user_id="user_1",
page=1,
page_size=10
)
assert result["total"] == 3
assert len(result["datasets"]) == 3
assert result["datasets"][0]["id"] == "dataset_1"
def test_evaluation_run_status_flow(self):
"""Test evaluation run status transitions"""
mock_service = Mock()
# Simulate status progression
statuses = ["PENDING", "RUNNING", "COMPLETED"]
for status in statuses:
mock_run = {"id": "run_123", "status": status}
mock_service.get_run_results.return_value = {"run": mock_run}
result = mock_service.get_run_results("run_123")
assert result["run"]["status"] == status
def test_bulk_import_success_count(self):
"""Test bulk import success/failure counting"""
mock_service = Mock()
# Simulate 8 successes, 2 failures
mock_service.import_test_cases.return_value = (8, 2)
success_count, failure_count = mock_service.import_test_cases(
dataset_id="dataset_123",
cases=[{"question": f"Q{i}"} for i in range(10)]
)
assert success_count == 8
assert failure_count == 2
assert success_count + failure_count == 10
def test_metrics_summary_aggregation(self):
"""Test metrics summary aggregation"""
results = [
{"metrics": {"precision": 0.9, "recall": 0.8}, "execution_time": 1.2},
{"metrics": {"precision": 0.8, "recall": 0.7}, "execution_time": 1.5},
{"metrics": {"precision": 0.85, "recall": 0.75}, "execution_time": 1.3}
]
# Calculate averages
avg_precision = sum(r["metrics"]["precision"] for r in results) / len(results)
avg_recall = sum(r["metrics"]["recall"] for r in results) / len(results)
avg_time = sum(r["execution_time"] for r in results) / len(results)
assert abs(avg_precision - 0.85) < 0.01
assert abs(avg_recall - 0.75) < 0.01
assert abs(avg_time - 1.33) < 0.01
def test_recommendation_severity_levels(self):
"""Test recommendation severity levels"""
severities = ["low", "medium", "high", "critical"]
for severity in severities:
rec = {
"issue": "Test Issue",
"severity": severity,
"suggestions": ["Fix it"]
}
assert rec["severity"] in severities
def test_empty_dataset_handling(self):
"""Test handling of empty datasets"""
mock_service = Mock()
mock_service.get_test_cases.return_value = []
cases = mock_service.get_test_cases("empty_dataset")
assert len(cases) == 0
assert isinstance(cases, list)
def test_error_handling(self):
"""Test error handling in service"""
mock_service = Mock()
mock_service.create_dataset.return_value = (False, "Dataset name cannot be empty")
success, error = mock_service.create_dataset(name="", kb_ids=[])
assert success is False
assert "empty" in error.lower()
def test_pagination_logic(self):
"""Test pagination logic"""
total_items = 50
page_size = 10
page = 2
# Calculate expected items for page 2
start = (page - 1) * page_size
end = min(start + page_size, total_items)
expected_count = end - start
assert expected_count == 10
assert start == 10
assert end == 20
class TestMetricsCalculations:
"""Test metric calculation logic"""
def test_precision_calculation(self):
"""Test precision calculation"""
retrieved = {"chunk_1", "chunk_2", "chunk_3", "chunk_4"}
relevant = {"chunk_1", "chunk_2", "chunk_5"}
precision = len(retrieved & relevant) / len(retrieved)
assert precision == 0.5 # 2 out of 4
def test_recall_calculation(self):
"""Test recall calculation"""
retrieved = {"chunk_1", "chunk_2", "chunk_3", "chunk_4"}
relevant = {"chunk_1", "chunk_2", "chunk_5"}
recall = len(retrieved & relevant) / len(relevant)
assert abs(recall - 0.67) < 0.01 # 2 out of 3
def test_hit_rate_positive(self):
"""Test hit rate when relevant chunk is found"""
retrieved = {"chunk_1", "chunk_2", "chunk_3"}
relevant = {"chunk_2", "chunk_4"}
hit_rate = 1.0 if (retrieved & relevant) else 0.0
assert hit_rate == 1.0
def test_hit_rate_negative(self):
"""Test hit rate when no relevant chunk is found"""
retrieved = {"chunk_1", "chunk_2", "chunk_3"}
relevant = {"chunk_4", "chunk_5"}
hit_rate = 1.0 if (retrieved & relevant) else 0.0
assert hit_rate == 0.0
def test_mrr_calculation(self):
"""Test MRR calculation"""
retrieved_ids = ["chunk_1", "chunk_2", "chunk_3", "chunk_4"]
relevant_ids = {"chunk_3", "chunk_5"}
mrr = 0.0
for i, chunk_id in enumerate(retrieved_ids, 1):
if chunk_id in relevant_ids:
mrr = 1.0 / i
break
assert abs(mrr - 0.33) < 0.01 # First relevant at position 3
# Summary test
def test_evaluation_framework_summary():
"""
Summary test to confirm all evaluation framework features work.
This test verifies that:
- Basic assertions work
- Mocking works for all service methods
- Metrics calculations are correct
- Error handling works
- Pagination logic works
"""
assert True, "Evaluation test framework is working correctly!"
if __name__ == "__main__":
pytest.main([__file__, "-v"])

View File

@ -1,557 +0,0 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Unit tests for RAG Evaluation Service
Tests cover:
- Dataset management (CRUD operations)
- Test case management
- Evaluation execution
- Metrics computation
- Recommendations generation
"""
import pytest
from unittest.mock import patch
class TestEvaluationDatasetManagement:
"""Tests for evaluation dataset management"""
@pytest.fixture
def mock_evaluation_service(self):
"""Create a mock EvaluationService"""
with patch('api.db.services.evaluation_service.EvaluationService') as mock:
yield mock
@pytest.fixture
def sample_dataset_data(self):
"""Sample dataset data for testing"""
return {
"name": "Customer Support QA",
"description": "Test cases for customer support",
"kb_ids": ["kb_123", "kb_456"],
"tenant_id": "tenant_1",
"user_id": "user_1"
}
def test_create_dataset_success(self, mock_evaluation_service, sample_dataset_data):
"""Test successful dataset creation"""
mock_evaluation_service.create_dataset.return_value = (True, "dataset_123")
success, dataset_id = mock_evaluation_service.create_dataset(**sample_dataset_data)
assert success is True
assert dataset_id == "dataset_123"
mock_evaluation_service.create_dataset.assert_called_once()
def test_create_dataset_with_empty_name(self, mock_evaluation_service):
"""Test dataset creation with empty name"""
data = {
"name": "",
"description": "Test",
"kb_ids": ["kb_123"],
"tenant_id": "tenant_1",
"user_id": "user_1"
}
mock_evaluation_service.create_dataset.return_value = (False, "Dataset name cannot be empty")
success, error = mock_evaluation_service.create_dataset(**data)
assert success is False
assert "name" in error.lower() or "empty" in error.lower()
def test_create_dataset_with_empty_kb_ids(self, mock_evaluation_service):
"""Test dataset creation with empty kb_ids"""
data = {
"name": "Test Dataset",
"description": "Test",
"kb_ids": [],
"tenant_id": "tenant_1",
"user_id": "user_1"
}
mock_evaluation_service.create_dataset.return_value = (False, "kb_ids cannot be empty")
success, error = mock_evaluation_service.create_dataset(**data)
assert success is False
def test_get_dataset_success(self, mock_evaluation_service):
"""Test successful dataset retrieval"""
expected_dataset = {
"id": "dataset_123",
"name": "Test Dataset",
"kb_ids": ["kb_123"]
}
mock_evaluation_service.get_dataset.return_value = expected_dataset
dataset = mock_evaluation_service.get_dataset("dataset_123")
assert dataset is not None
assert dataset["id"] == "dataset_123"
def test_get_dataset_not_found(self, mock_evaluation_service):
"""Test getting non-existent dataset"""
mock_evaluation_service.get_dataset.return_value = None
dataset = mock_evaluation_service.get_dataset("nonexistent")
assert dataset is None
def test_list_datasets(self, mock_evaluation_service):
"""Test listing datasets"""
expected_result = {
"total": 2,
"datasets": [
{"id": "dataset_1", "name": "Dataset 1"},
{"id": "dataset_2", "name": "Dataset 2"}
]
}
mock_evaluation_service.list_datasets.return_value = expected_result
result = mock_evaluation_service.list_datasets(
tenant_id="tenant_1",
user_id="user_1",
page=1,
page_size=20
)
assert result["total"] == 2
assert len(result["datasets"]) == 2
def test_list_datasets_with_pagination(self, mock_evaluation_service):
"""Test listing datasets with pagination"""
mock_evaluation_service.list_datasets.return_value = {
"total": 50,
"datasets": [{"id": f"dataset_{i}"} for i in range(10)]
}
result = mock_evaluation_service.list_datasets(
tenant_id="tenant_1",
user_id="user_1",
page=2,
page_size=10
)
assert result["total"] == 50
assert len(result["datasets"]) == 10
def test_update_dataset_success(self, mock_evaluation_service):
"""Test successful dataset update"""
mock_evaluation_service.update_dataset.return_value = True
success = mock_evaluation_service.update_dataset(
"dataset_123",
name="Updated Name",
description="Updated Description"
)
assert success is True
def test_update_dataset_not_found(self, mock_evaluation_service):
"""Test updating non-existent dataset"""
mock_evaluation_service.update_dataset.return_value = False
success = mock_evaluation_service.update_dataset(
"nonexistent",
name="Updated Name"
)
assert success is False
def test_delete_dataset_success(self, mock_evaluation_service):
"""Test successful dataset deletion"""
mock_evaluation_service.delete_dataset.return_value = True
success = mock_evaluation_service.delete_dataset("dataset_123")
assert success is True
def test_delete_dataset_not_found(self, mock_evaluation_service):
"""Test deleting non-existent dataset"""
mock_evaluation_service.delete_dataset.return_value = False
success = mock_evaluation_service.delete_dataset("nonexistent")
assert success is False
class TestEvaluationTestCaseManagement:
"""Tests for test case management"""
@pytest.fixture
def mock_evaluation_service(self):
"""Create a mock EvaluationService"""
with patch('api.db.services.evaluation_service.EvaluationService') as mock:
yield mock
@pytest.fixture
def sample_test_case(self):
"""Sample test case data"""
return {
"dataset_id": "dataset_123",
"question": "How do I reset my password?",
"reference_answer": "Click on 'Forgot Password' and follow the email instructions.",
"relevant_doc_ids": ["doc_789"],
"relevant_chunk_ids": ["chunk_101", "chunk_102"]
}
def test_add_test_case_success(self, mock_evaluation_service, sample_test_case):
"""Test successful test case addition"""
mock_evaluation_service.add_test_case.return_value = (True, "case_123")
success, case_id = mock_evaluation_service.add_test_case(**sample_test_case)
assert success is True
assert case_id == "case_123"
def test_add_test_case_with_empty_question(self, mock_evaluation_service):
"""Test adding test case with empty question"""
mock_evaluation_service.add_test_case.return_value = (False, "Question cannot be empty")
success, error = mock_evaluation_service.add_test_case(
dataset_id="dataset_123",
question=""
)
assert success is False
assert "question" in error.lower() or "empty" in error.lower()
def test_add_test_case_without_reference_answer(self, mock_evaluation_service):
"""Test adding test case without reference answer (optional)"""
mock_evaluation_service.add_test_case.return_value = (True, "case_123")
success, case_id = mock_evaluation_service.add_test_case(
dataset_id="dataset_123",
question="Test question",
reference_answer=None
)
assert success is True
def test_get_test_cases(self, mock_evaluation_service):
"""Test getting all test cases for a dataset"""
expected_cases = [
{"id": "case_1", "question": "Question 1"},
{"id": "case_2", "question": "Question 2"}
]
mock_evaluation_service.get_test_cases.return_value = expected_cases
cases = mock_evaluation_service.get_test_cases("dataset_123")
assert len(cases) == 2
assert cases[0]["id"] == "case_1"
def test_get_test_cases_empty_dataset(self, mock_evaluation_service):
"""Test getting test cases from empty dataset"""
mock_evaluation_service.get_test_cases.return_value = []
cases = mock_evaluation_service.get_test_cases("dataset_123")
assert len(cases) == 0
def test_delete_test_case_success(self, mock_evaluation_service):
"""Test successful test case deletion"""
mock_evaluation_service.delete_test_case.return_value = True
success = mock_evaluation_service.delete_test_case("case_123")
assert success is True
def test_import_test_cases_success(self, mock_evaluation_service):
"""Test bulk import of test cases"""
cases = [
{"question": "Question 1", "reference_answer": "Answer 1"},
{"question": "Question 2", "reference_answer": "Answer 2"},
{"question": "Question 3", "reference_answer": "Answer 3"}
]
mock_evaluation_service.import_test_cases.return_value = (3, 0)
success_count, failure_count = mock_evaluation_service.import_test_cases(
"dataset_123",
cases
)
assert success_count == 3
assert failure_count == 0
def test_import_test_cases_with_failures(self, mock_evaluation_service):
"""Test bulk import with some failures"""
cases = [
{"question": "Question 1"},
{"question": ""}, # Invalid
{"question": "Question 3"}
]
mock_evaluation_service.import_test_cases.return_value = (2, 1)
success_count, failure_count = mock_evaluation_service.import_test_cases(
"dataset_123",
cases
)
assert success_count == 2
assert failure_count == 1
class TestEvaluationExecution:
"""Tests for evaluation execution"""
@pytest.fixture
def mock_evaluation_service(self):
"""Create a mock EvaluationService"""
with patch('api.db.services.evaluation_service.EvaluationService') as mock:
yield mock
def test_start_evaluation_success(self, mock_evaluation_service):
"""Test successful evaluation start"""
mock_evaluation_service.start_evaluation.return_value = (True, "run_123")
success, run_id = mock_evaluation_service.start_evaluation(
dataset_id="dataset_123",
dialog_id="dialog_456",
user_id="user_1"
)
assert success is True
assert run_id == "run_123"
def test_start_evaluation_with_invalid_dialog(self, mock_evaluation_service):
"""Test starting evaluation with invalid dialog"""
mock_evaluation_service.start_evaluation.return_value = (False, "Dialog not found")
success, error = mock_evaluation_service.start_evaluation(
dataset_id="dataset_123",
dialog_id="nonexistent",
user_id="user_1"
)
assert success is False
assert "dialog" in error.lower()
def test_start_evaluation_with_custom_name(self, mock_evaluation_service):
"""Test starting evaluation with custom name"""
mock_evaluation_service.start_evaluation.return_value = (True, "run_123")
success, run_id = mock_evaluation_service.start_evaluation(
dataset_id="dataset_123",
dialog_id="dialog_456",
user_id="user_1",
name="My Custom Evaluation"
)
assert success is True
def test_get_run_results(self, mock_evaluation_service):
"""Test getting evaluation run results"""
expected_results = {
"run": {
"id": "run_123",
"status": "COMPLETED",
"metrics_summary": {
"avg_precision": 0.85,
"avg_recall": 0.78
}
},
"results": [
{"case_id": "case_1", "metrics": {"precision": 0.9}},
{"case_id": "case_2", "metrics": {"precision": 0.8}}
]
}
mock_evaluation_service.get_run_results.return_value = expected_results
results = mock_evaluation_service.get_run_results("run_123")
assert results["run"]["id"] == "run_123"
assert len(results["results"]) == 2
def test_get_run_results_not_found(self, mock_evaluation_service):
"""Test getting results for non-existent run"""
mock_evaluation_service.get_run_results.return_value = {}
results = mock_evaluation_service.get_run_results("nonexistent")
assert results == {}
class TestEvaluationMetrics:
"""Tests for metrics computation"""
@pytest.fixture
def mock_evaluation_service(self):
"""Create a mock EvaluationService"""
with patch('api.db.services.evaluation_service.EvaluationService') as mock:
yield mock
def test_compute_retrieval_metrics_perfect_match(self, mock_evaluation_service):
"""Test retrieval metrics with perfect match"""
retrieved_ids = ["chunk_1", "chunk_2", "chunk_3"]
relevant_ids = ["chunk_1", "chunk_2", "chunk_3"]
expected_metrics = {
"precision": 1.0,
"recall": 1.0,
"f1_score": 1.0,
"hit_rate": 1.0,
"mrr": 1.0
}
mock_evaluation_service._compute_retrieval_metrics.return_value = expected_metrics
metrics = mock_evaluation_service._compute_retrieval_metrics(retrieved_ids, relevant_ids)
assert metrics["precision"] == 1.0
assert metrics["recall"] == 1.0
assert metrics["f1_score"] == 1.0
def test_compute_retrieval_metrics_partial_match(self, mock_evaluation_service):
"""Test retrieval metrics with partial match"""
retrieved_ids = ["chunk_1", "chunk_2", "chunk_4", "chunk_5"]
relevant_ids = ["chunk_1", "chunk_2", "chunk_3"]
expected_metrics = {
"precision": 0.5, # 2 out of 4 retrieved are relevant
"recall": 0.67, # 2 out of 3 relevant were retrieved
"f1_score": 0.57,
"hit_rate": 1.0, # At least one relevant was retrieved
"mrr": 1.0 # First retrieved is relevant
}
mock_evaluation_service._compute_retrieval_metrics.return_value = expected_metrics
metrics = mock_evaluation_service._compute_retrieval_metrics(retrieved_ids, relevant_ids)
assert metrics["precision"] < 1.0
assert metrics["recall"] < 1.0
assert metrics["hit_rate"] == 1.0
def test_compute_retrieval_metrics_no_match(self, mock_evaluation_service):
"""Test retrieval metrics with no match"""
retrieved_ids = ["chunk_4", "chunk_5", "chunk_6"]
relevant_ids = ["chunk_1", "chunk_2", "chunk_3"]
expected_metrics = {
"precision": 0.0,
"recall": 0.0,
"f1_score": 0.0,
"hit_rate": 0.0,
"mrr": 0.0
}
mock_evaluation_service._compute_retrieval_metrics.return_value = expected_metrics
metrics = mock_evaluation_service._compute_retrieval_metrics(retrieved_ids, relevant_ids)
assert metrics["precision"] == 0.0
assert metrics["recall"] == 0.0
assert metrics["hit_rate"] == 0.0
def test_compute_summary_metrics(self, mock_evaluation_service):
"""Test summary metrics computation"""
results = [
{"metrics": {"precision": 0.9, "recall": 0.8}, "execution_time": 1.2},
{"metrics": {"precision": 0.8, "recall": 0.7}, "execution_time": 1.5},
{"metrics": {"precision": 0.85, "recall": 0.75}, "execution_time": 1.3}
]
expected_summary = {
"total_cases": 3,
"avg_execution_time": 1.33,
"avg_precision": 0.85,
"avg_recall": 0.75
}
mock_evaluation_service._compute_summary_metrics.return_value = expected_summary
summary = mock_evaluation_service._compute_summary_metrics(results)
assert summary["total_cases"] == 3
assert summary["avg_precision"] > 0.8
class TestEvaluationRecommendations:
"""Tests for configuration recommendations"""
@pytest.fixture
def mock_evaluation_service(self):
"""Create a mock EvaluationService"""
with patch('api.db.services.evaluation_service.EvaluationService') as mock:
yield mock
def test_get_recommendations_low_precision(self, mock_evaluation_service):
"""Test recommendations for low precision"""
recommendations = [
{
"issue": "Low Precision",
"severity": "high",
"suggestions": [
"Increase similarity_threshold",
"Enable reranking"
]
}
]
mock_evaluation_service.get_recommendations.return_value = recommendations
recs = mock_evaluation_service.get_recommendations("run_123")
assert len(recs) > 0
assert any("precision" in r["issue"].lower() for r in recs)
def test_get_recommendations_low_recall(self, mock_evaluation_service):
"""Test recommendations for low recall"""
recommendations = [
{
"issue": "Low Recall",
"severity": "high",
"suggestions": [
"Increase top_k",
"Lower similarity_threshold"
]
}
]
mock_evaluation_service.get_recommendations.return_value = recommendations
recs = mock_evaluation_service.get_recommendations("run_123")
assert len(recs) > 0
assert any("recall" in r["issue"].lower() for r in recs)
def test_get_recommendations_slow_response(self, mock_evaluation_service):
"""Test recommendations for slow response time"""
recommendations = [
{
"issue": "Slow Response Time",
"severity": "medium",
"suggestions": [
"Reduce top_k",
"Optimize embedding model"
]
}
]
mock_evaluation_service.get_recommendations.return_value = recommendations
recs = mock_evaluation_service.get_recommendations("run_123")
assert len(recs) > 0
assert any("response" in r["issue"].lower() or "slow" in r["issue"].lower() for r in recs)
def test_get_recommendations_no_issues(self, mock_evaluation_service):
"""Test recommendations when metrics are good"""
mock_evaluation_service.get_recommendations.return_value = []
recs = mock_evaluation_service.get_recommendations("run_123")
assert len(recs) == 0
if __name__ == "__main__":
pytest.main([__file__, "-v"])

View File

@ -217,20 +217,23 @@ const MarkdownContent = ({
const docType = chunkItem?.doc_type;
return showImage(docType) ? (
<Image
id={imageId}
className={styles.referenceInnerChunkImage}
onClick={
documentId
? handleDocumentButtonClick(
documentId,
chunkItem,
fileExtension === 'pdf',
documentUrl,
)
: () => {}
}
></Image>
<section>
<Image
id={imageId}
className={styles.referenceInnerChunkImage}
onClick={
documentId
? handleDocumentButtonClick(
documentId,
chunkItem,
fileExtension === 'pdf',
documentUrl,
)
: () => {}
}
></Image>
<span className="text-accent-primary"> {imageId}</span>
</section>
) : (
<HoverCard key={i}>
<HoverCardTrigger>

View File

@ -220,20 +220,23 @@ function MarkdownContent({
const docType = chunkItem?.doc_type;
return showImage(docType) ? (
<Image
id={imageId}
className={styles.referenceInnerChunkImage}
onClick={
documentId
? handleDocumentButtonClick(
documentId,
chunkItem,
fileExtension === 'pdf',
documentUrl,
)
: () => {}
}
></Image>
<section>
<Image
id={imageId}
className={styles.referenceInnerChunkImage}
onClick={
documentId
? handleDocumentButtonClick(
documentId,
chunkItem,
fileExtension === 'pdf',
documentUrl,
)
: () => {}
}
></Image>
<span className="text-accent-primary">{imageId}</span>
</section>
) : (
<HoverCard key={i}>
<HoverCardTrigger>

View File

@ -0,0 +1,191 @@
import { useEffect, useMemo } from 'react';
import { ControllerRenderProps, useFormContext } from 'react-hook-form';
import { Checkbox } from '@/components/ui/checkbox';
import { Input } from '@/components/ui/input';
import { cn } from '@/lib/utils';
/* ---------------- Token Field ---------------- */
export type ConfluenceTokenFieldProps = ControllerRenderProps & {
fieldType: 'username' | 'token';
placeholder?: string;
disabled?: boolean;
};
const ConfluenceTokenField = ({
fieldType,
value,
onChange,
placeholder,
disabled,
...rest
}: ConfluenceTokenFieldProps) => {
return (
<div className="flex w-full flex-col gap-2">
<Input
className="w-full"
type={fieldType === 'token' ? 'password' : 'text'}
value={value ?? ''}
onChange={(e) => onChange(e.target.value)}
placeholder={
placeholder ||
(fieldType === 'token'
? 'Enter your Confluence access token'
: 'Confluence username or email')
}
disabled={disabled}
{...rest}
/>
</div>
);
};
/* ---------------- Indexing Mode Field ---------------- */
type ConfluenceIndexingMode = 'everything' | 'space' | 'page';
export type ConfluenceIndexingModeFieldProps = ControllerRenderProps;
export const ConfluenceIndexingModeField = (
fieldProps: ConfluenceIndexingModeFieldProps,
) => {
const { value, onChange, disabled } = fieldProps;
const { watch, setValue } = useFormContext();
const mode = useMemo<ConfluenceIndexingMode>(
() => (value as ConfluenceIndexingMode) || 'everything',
[value],
);
const spaceValue = watch('config.space');
const pageIdValue = watch('config.page_id');
const indexRecursively = watch('config.index_recursively');
useEffect(() => {
if (!value) onChange('everything');
}, [value, onChange]);
const handleModeChange = (nextMode?: string) => {
const normalized = (nextMode || 'everything') as ConfluenceIndexingMode;
onChange(normalized);
if (normalized === 'everything') {
setValue('config.space', '', { shouldDirty: true, shouldTouch: true });
setValue('config.page_id', '', { shouldDirty: true, shouldTouch: true });
setValue('config.index_recursively', false, {
shouldDirty: true,
shouldTouch: true,
});
} else if (normalized === 'space') {
setValue('config.page_id', '', { shouldDirty: true, shouldTouch: true });
setValue('config.index_recursively', false, {
shouldDirty: true,
shouldTouch: true,
});
} else if (normalized === 'page') {
setValue('config.space', '', { shouldDirty: true, shouldTouch: true });
}
};
return (
<div className="w-full rounded-lg border border-border-button bg-bg-card p-4 space-y-4">
<div className="flex items-center gap-2 text-sm font-medium text-text-secondary">
{INDEX_MODE_OPTIONS.map((option) => {
const isActive = option.value === mode;
return (
<button
key={option.value}
type="button"
disabled={disabled}
onClick={() => handleModeChange(option.value)}
className={cn(
'flex-1 rounded-lg border px-3 py-2 transition-all',
'border-transparent bg-transparent text-text-secondary hover:border-border-button hover:bg-bg-card-secondary',
isActive &&
'border-border-button bg-background text-primary shadow-sm',
)}
>
{option.label}
</button>
);
})}
</div>
{mode === 'everything' && (
<p className="text-sm text-text-secondary">
This connector will index all pages the provided credentials have
access to.
</p>
)}
{mode === 'space' && (
<div className="space-y-2">
<div className="text-sm font-semibold text-text-primary">
Space Key
</div>
<Input
className="w-full"
value={spaceValue ?? ''}
onChange={(e) =>
setValue('config.space', e.target.value, {
shouldDirty: true,
shouldTouch: true,
})
}
placeholder="e.g. KB"
disabled={disabled}
/>
<p className="text-xs text-text-secondary">
The Confluence space key to index.
</p>
</div>
)}
{mode === 'page' && (
<div className="space-y-2">
<div className="text-sm font-semibold text-text-primary">Page ID</div>
<Input
className="w-full"
value={pageIdValue ?? ''}
onChange={(e) =>
setValue('config.page_id', e.target.value, {
shouldDirty: true,
shouldTouch: true,
})
}
placeholder="e.g. 123456"
disabled={disabled}
/>
<p className="text-xs text-text-secondary">
The Confluence page ID to index.
</p>
<div className="flex items-center gap-2 pt-2">
<Checkbox
checked={Boolean(indexRecursively)}
onCheckedChange={(checked) =>
setValue('config.index_recursively', Boolean(checked), {
shouldDirty: true,
shouldTouch: true,
})
}
disabled={disabled}
/>
<span className="text-sm text-text-secondary">
Index child pages recursively
</span>
</div>
</div>
)}
</div>
);
};
const INDEX_MODE_OPTIONS = [
{ label: 'Everything', value: 'everything' },
{ label: 'Space', value: 'space' },
{ label: 'Page', value: 'page' },
];
export default ConfluenceTokenField;

View File

@ -1,9 +1,9 @@
import { FormFieldType } from '@/components/dynamic-form';
import SvgIcon from '@/components/svg-icon';
import { t } from 'i18next';
import { ConfluenceIndexingModeField } from './component/confluence-token-field';
import GmailTokenField from './component/gmail-token-field';
import GoogleDriveTokenField from './component/google-drive-token-field';
export enum DataSourceKey {
CONFLUENCE = 'confluence',
S3 = 's3',
@ -230,12 +230,35 @@ export const DataSourceFormFields = {
required: false,
tooltip: t('setting.confluenceIsCloudTip'),
},
{
label: 'Index Method',
name: 'config.index_mode',
type: FormFieldType.Text, // keep as text so RHF registers it
required: false,
horizontal: true,
labelClassName: 'self-start pt-4',
render: (fieldProps) => <ConfluenceIndexingModeField {...fieldProps} />,
},
{
label: 'Space Key',
name: 'config.space',
type: FormFieldType.Text,
required: false,
tooltip: t('setting.confluenceSpaceKeyTip'),
hidden: true,
},
{
label: 'Page ID',
name: 'config.page_id',
type: FormFieldType.Text,
required: false,
hidden: true,
},
{
label: 'Index Recursively',
name: 'config.index_recursively',
type: FormFieldType.Checkbox,
required: false,
hidden: true,
},
],
[DataSourceKey.GOOGLE_DRIVE]: [