Feat: support cross-lang search. (#7557)

### What problem does this PR solve?

#7376
#4503
#5710 
#7470

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

api/apps/chunk_app.py

@@ -22,7 +22,7 @@ from flask_login import login_required, current_user
 from rag.app.qa import rmPrefix, beAdoc
 from rag.app.tag import label_question
 from rag.nlp import search, rag_tokenizer
-from rag.prompts import keyword_extraction
+from rag.prompts import keyword_extraction, cross_languages
 from rag.settings import PAGERANK_FLD
 from rag.utils import rmSpace
 from api.db import LLMType, ParserType
@@ -275,6 +275,7 @@ def retrieval_test():
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
     use_kg = req.get("use_kg", False)
     top = int(req.get("top_k", 1024))
+    langs = req.get("cross_languages", [])
     tenant_ids = []
 
     try:
@@ -294,6 +295,9 @@ def retrieval_test():
         if not e:
             return get_data_error_result(message="Knowledgebase not found!")
 
+        if langs:
+            question = cross_languages(kb.tenant_id, None, question, langs)
+
         embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
 
         rerank_mdl = None

api/db/services/dialog_service.py

@@ -36,7 +36,8 @@ from api.utils import current_timestamp, datetime_format
 from rag.app.resume import forbidden_select_fields4resume
 from rag.app.tag import label_question
 from rag.nlp.search import index_name
-from rag.prompts import chunks_format, citation_prompt, full_question, kb_prompt, keyword_extraction, llm_id2llm_type, message_fit_in
+from rag.prompts import chunks_format, citation_prompt, full_question, kb_prompt, keyword_extraction, llm_id2llm_type, message_fit_in, \
+    cross_languages
 from rag.utils import num_tokens_from_string, rmSpace
 from rag.utils.tavily_conn import Tavily
 
@@ -214,6 +215,9 @@ def chat(dialog, messages, stream=True, **kwargs):
     else:
         questions = questions[-1:]
 
+    if prompt_config.get("cross_languages"):
+        questions = [cross_languages(dialog.tenant_id, dialog.llm_id, questions[0], prompt_config["cross_languages"])]
+
     refine_question_ts = timer()
 
     rerank_mdl = None

api/db/services/document_service.py

@@ -131,7 +131,7 @@ class DocumentService(CommonService):
         if types:
             query = query.where(cls.model.type.in_(types))
 
-        return query.scalar() or 0
+        return int(query.scalar()) or 0
 
     @classmethod
     @DB.connection_context()