Fix: Fixed the issue where tag content would overflow the container #8392 (#8393)

### What problem does this PR solve?

Fix: Fixed the issue where tag content would overflow the container
#8392
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

.github/workflows/tests.yml

@@ -88,7 +88,12 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && uv sync --python 3.10 --group test --frozen && uv pip install . && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python && uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
 
       - name: Run frontend api tests against Elasticsearch
         run: |
@@ -98,7 +103,7 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
+          cd sdk/python && UV_LINK_MODE=copy uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
           
       - name: Run http api tests against Elasticsearch
         run: |
@@ -113,7 +118,7 @@ jobs:
           else
             export HTTP_API_TEST_LEVEL=p2
           fi
-          cd sdk/python && uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_http_api && pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL}
+          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
@@ -132,7 +137,12 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && uv sync --python 3.10 --group test --frozen && uv pip install . && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python && DOC_ENGINE=infinity uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
 
       - name: Run frontend api tests against Infinity
         run: |
@@ -142,7 +152,7 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
+          cd sdk/python && UV_LINK_MODE=copy uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
 
       - name: Run http api tests against Infinity
         run: |
@@ -157,7 +167,7 @@ jobs:
           else
             export HTTP_API_TEST_LEVEL=p2
           fi
-          cd sdk/python && uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_http_api && DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL}
+          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && DOC_ENGINE=infinity uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed

.gitignore

@@ -36,6 +36,12 @@ sdk/python/ragflow.egg-info/
 sdk/python/build/
 sdk/python/dist/
 sdk/python/ragflow_sdk.egg-info/
+
+# Exclude dep files
+libssl*.deb
+tika-server*.jar*
+cl100k_base.tiktoken
+chrome*
 huggingface.co/
 nltk_data/
 

README.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DBEDFA"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,6 +30,9 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
@@ -40,6 +43,12 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
+
 <details open>
 <summary><b>📕 Table of Contents</b></summary>
 
@@ -78,11 +87,11 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Latest Updates
 
+- 2025-05-23 Adds a Python/JavaScript code executor component to Agent.
+- 2025-05-05 Supports cross-language query.
 - 2025-03-19 Supports using a multi-modal model to make sense of images within PDF or DOCX files.
 - 2025-02-28 Combined with Internet search (Tavily), supports reasoning like Deep Research for any LLMs.
-- 2025-01-26 Optimizes knowledge graph extraction and application, offering various configuration options.
 - 2024-12-18 Upgrades Document Layout Analysis model in DeepDoc.
-- 2024-11-01 Adds keyword extraction and related question generation to the parsed chunks to improve the accuracy of retrieval.
 - 2024-08-22 Support text to SQL statements through RAG.
 
 ## 🎉 Stay Tuned
@@ -178,7 +187,7 @@ releases! 🌟
 > All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
 > If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
 
-   > The command below downloads the `v0.19.0-slim` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.19.0-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0` for the full edition `v0.19.0`.
+   > The command below downloads the `v0.19.1-slim` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.19.1-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` for the full edition `v0.19.1`.
 
    ```bash
    $ cd ragflow/docker
@@ -191,8 +200,8 @@ releases! 🌟
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    |-------------------|-----------------|-----------------------|--------------------------|
-   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.19.0-slim      | &approx;2       | ❌                   | Stable release            |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                   | Stable release            |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
    | nightly-slim      | &approx;2       | ❌                   | _Unstable_ nightly build  |
 

README_id.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體中文版自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DBEDFA"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Lencana Daring" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Rilis%20Terbaru" alt="Rilis Terbaru">
@@ -30,6 +30,9 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/Lisensi-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="Lisensi">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
@@ -40,6 +43,8 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
 <details open>
 <summary><b>📕 Daftar Isi </b> </summary>
 
@@ -75,11 +80,11 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Pembaruan Terbaru
 
+- 2025-05-23 Menambahkan komponen pelaksana kode Python/JS ke Agen.
+- 2025-05-05 Mendukung kueri lintas bahasa.
 - 2025-03-19 Mendukung penggunaan model multi-modal untuk memahami gambar di dalam file PDF atau DOCX.
 - 2025-02-28 dikombinasikan dengan pencarian Internet (TAVILY), mendukung penelitian mendalam untuk LLM apa pun.
-- 2025-01-26 Optimalkan ekstraksi dan penerapan grafik pengetahuan dan sediakan berbagai opsi konfigurasi.
 - 2024-12-18 Meningkatkan model Analisis Tata Letak Dokumen di DeepDoc.
-- 2024-11-01 Penambahan ekstraksi kata kunci dan pembuatan pertanyaan terkait untuk meningkatkan akurasi pengambilan.
 - 2024-08-22 Dukungan untuk teks ke pernyataan SQL melalui RAG.
 
 ## 🎉 Tetap Terkini
@@ -173,7 +178,7 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 > Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
 > Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
 
-> Perintah di bawah ini mengunduh edisi v0.19.0-slim dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.19.0-slim, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server. Misalnya, atur RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0 untuk edisi lengkap v0.19.0.
+> Perintah di bawah ini mengunduh edisi v0.19.1-slim dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.19.1-slim, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server. Misalnya, atur RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1 untuk edisi lengkap v0.19.1.
 
 ```bash
 $ cd ragflow/docker
@@ -186,8 +191,8 @@ $ docker compose -f docker-compose.yml up -d
 
 | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
 | ----------------- | --------------- | --------------------- | ------------------------ |
-| v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-| v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+| v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+| v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
 | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
 | nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
 

README_ja.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體中文版自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DBEDFA"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,6 +30,9 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
@@ -40,6 +43,8 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
 ## 💡 RAGFlow とは？
 
 [RAGFlow](https://ragflow.io/) は、深い文書理解に基づいたオープンソースの RAG (Retrieval-Augmented Generation) エンジンである。LLM（大規模言語モデル）を組み合わせることで、様々な複雑なフォーマットのデータから根拠のある引用に裏打ちされた、信頼できる質問応答機能を実現し、あらゆる規模のビジネスに適した RAG ワークフローを提供します。
@@ -55,11 +60,11 @@
 
 ## 🔥 最新情報
 
+- 2025-05-23 エージェントに Python/JS コードエグゼキュータコンポーネントを追加しました。
+- 2025-05-05 言語間クエリをサポートしました。
 - 2025-03-19 PDFまたはDOCXファイル内の画像を理解するために、多モーダルモデルを使用することをサポートします。
 - 2025-02-28 インターネット検索 (TAVILY) と組み合わせて、あらゆる LLM の詳細な調査をサポートします。
-- 2025-01-26 ナレッジ グラフの抽出と適用を最適化し、さまざまな構成オプションを提供します。
 - 2024-12-18 DeepDoc のドキュメント レイアウト分析モデルをアップグレードします。
-- 2024-11-01 再現の精度を向上させるために、解析されたチャンクにキーワード抽出と関連質問の生成を追加しました。
 - 2024-08-22 RAG を介して SQL ステートメントへのテキストをサポートします。
 
 ## 🎉 続きを楽しみに
@@ -152,7 +157,7 @@
 > 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
 > ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
 
-   > 以下のコマンドは、RAGFlow Docker イメージの v0.19.0-slim エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.19.0-slim とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。例えば、完全版 v0.19.0 をダウンロードするには、RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0 と設定します。
+   > 以下のコマンドは、RAGFlow Docker イメージの v0.19.1-slim エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.19.1-slim とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。例えば、完全版 v0.19.1 をダウンロードするには、RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1 と設定します。
 
    ```bash
    $ cd ragflow/docker
@@ -165,8 +170,8 @@
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
    | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 

README_ko.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DBEDFA"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,6 +30,9 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
@@ -40,6 +43,8 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
 ## 💡 RAGFlow란?
 
 [RAGFlow](https://ragflow.io/)는 심층 문서 이해에 기반한 오픈소스 RAG (Retrieval-Augmented Generation) 엔진입니다. 이 엔진은 대규모 언어 모델(LLM)과 결합하여 정확한 질문 응답 기능을 제공하며, 다양한 복잡한 형식의 데이터에서 신뢰할 수 있는 출처를 바탕으로 한 인용을 통해 이를 뒷받침합니다. RAGFlow는 규모에 상관없이 모든 기업에 최적화된 RAG 워크플로우를 제공합니다.
@@ -55,11 +60,11 @@
 
 ## 🔥 업데이트
 
+- 2025-05-23 Agent에 Python/JS 코드 실행기 구성 요소를 추가합니다.
+- 2025-05-05 언어 간 쿼리를 지원합니다.
 - 2025-03-19 PDF 또는 DOCX 파일 내의 이미지를 이해하기 위해 다중 모드 모델을 사용하는 것을 지원합니다.
 - 2025-02-28 인터넷 검색(TAVILY)과 결합되어 모든 LLM에 대한 심층 연구를 지원합니다.
-- 2025-01-26 지식 그래프 추출 및 적용을 최적화하고 다양한 구성 옵션을 제공합니다.
 - 2024-12-18 DeepDoc의 문서 레이아웃 분석 모델 업그레이드.
-- 2024-11-01 파싱된 청크에 키워드 추출 및 관련 질문 생성을 추가하여 재현율을 향상시킵니다.
 - 2024-08-22 RAG를 통해 SQL 문에 텍스트를 지원합니다.
 
 ## 🎉 계속 지켜봐 주세요
@@ -152,7 +157,7 @@
 > 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
 > ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
 
-   > 아래 명령어는 RAGFlow Docker 이미지의 v0.19.0-slim 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.19.0-slim과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오. 예를 들어, 전체 버전인 v0.19.0을 다운로드하려면 RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0로 설정합니다.
+   > 아래 명령어는 RAGFlow Docker 이미지의 v0.19.1-slim 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.19.1-slim과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오. 예를 들어, 전체 버전인 v0.19.1을 다운로드하려면 RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1로 설정합니다.
 
    ```bash
    $ cd ragflow/docker
@@ -165,8 +170,8 @@
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
    | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 

README_pt_br.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DBEDFA"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Badge Estático" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Última%20Relese" alt="Última Versão">
@@ -30,6 +30,9 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="licença">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
@@ -40,6 +43,8 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
 <details open>
 <summary><b>📕 Índice</b></summary>
 
@@ -75,11 +80,11 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Últimas Atualizações
 
+- 23-05-2025 Adicione o componente executor de código Python/JS ao Agente.
+- 05-05-2025 Suporte a consultas entre idiomas.
 - 19-03-2025 Suporta o uso de um modelo multi-modal para entender imagens dentro de arquivos PDF ou DOCX.
 - 28-02-2025 combinado com a pesquisa na Internet (T AVI LY), suporta pesquisas profundas para qualquer LLM.
-- 26-01-2025 Otimize a extração e aplicação de gráficos de conhecimento e forneça uma variedade de opções de configuração.
 - 18-12-2024 Atualiza o modelo de Análise de Layout de Documentos no DeepDoc.
-- 01-11-2024 Adiciona extração de palavras-chave e geração de perguntas relacionadas aos blocos analisados para melhorar a precisão da recuperação.
 - 22-08-2024 Suporta conversão de texto para comandos SQL via RAG.
 
 ## 🎉 Fique Ligado
@@ -172,7 +177,7 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 > Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
 > Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
 
-    > O comando abaixo baixa a edição `v0.19.0-slim` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.19.0-slim`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor. Por exemplo: defina `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0` para a edição completa `v0.19.0`.
+    > O comando abaixo baixa a edição `v0.19.1-slim` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.19.1-slim`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor. Por exemplo: defina `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` para a edição completa `v0.19.1`.
 
     ```bash
     $ cd ragflow/docker
@@ -185,8 +190,8 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
     | Tag da imagem RAGFlow | Tamanho da imagem (GB) | Possui modelos de incorporação? | Estável?                 |
     | --------------------- | ---------------------- | ------------------------------- | ------------------------ |
-    | v0.19.0               | ~9                     | :heavy_check_mark:              | Lançamento estável       |
-    | v0.19.0-slim          | ~2                     | ❌                              | Lançamento estável       |
+    | v0.19.1               | ~9                     | :heavy_check_mark:              | Lançamento estável       |
+    | v0.19.1-slim          | ~2                     | ❌                              | Lançamento estável       |
     | nightly               | ~9                     | :heavy_check_mark:              | _Instável_ build noturno |
     | nightly-slim          | ~2                     | ❌                               | _Instável_ build noturno |
 

README_tzh.md

@@ -5,12 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DBEDFA"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -21,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -29,6 +30,9 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
@@ -39,6 +43,31 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
+
+<details open>
+<summary><b>📕 目錄</b></summary>
+
+- 💡 [RAGFlow 是什麼？](#-RAGFlow-是什麼)
+- 🎮 [Demo-試用](#-demo-試用)
+- 📌 [近期更新](#-近期更新)
+- 🌟 [主要功能](#-主要功能)
+- 🔎 [系統架構](#-系統架構)
+- 🎬 [快速開始](#-快速開始)
+- 🔧 [系統配置](#-系統配置)
+- 🔨 [以原始碼啟動服務](#-以原始碼啟動服務)
+- 📚 [技術文檔](#-技術文檔)
+- 📜 [路線圖](#-路線圖)
+- 🏄 [貢獻指南](#-貢獻指南)
+- 🙌 [加入社區](#-加入社區)
+- 🤝 [商務合作](#-商務合作)
+
+</details>
+
 ## 💡 RAGFlow 是什麼？
 
 [RAGFlow](https://ragflow.io/) 是一款基於深度文件理解所建構的開源 RAG（Retrieval-Augmented Generation）引擎。 RAGFlow 可以為各種規模的企業及個人提供一套精簡的 RAG 工作流程，結合大語言模型（LLM）針對用戶各類不同的複雜格式數據提供可靠的問答以及有理有據的引用。
@@ -54,11 +83,11 @@
 
 ## 🔥 近期更新
 
+- 2025-05-23 為 Agent 新增 Python/JS 程式碼執行器元件。
+- 2025-05-05 支援跨語言查詢。
 - 2025-03-19 PDF和DOCX中的圖支持用多模態大模型去解析得到描述.
 - 2025-02-28 結合網路搜尋（Tavily），對於任意大模型實現類似 Deep Research 的推理功能.
-- 2025-01-26 最佳化知識圖譜的擷取與應用，提供了多種配置選擇。
 - 2024-12-18 升級了 DeepDoc 的文檔佈局分析模型。
-- 2024-11-01 對解析後的 chunk 加入關鍵字抽取和相關問題產生以提高回想的準確度。
 - 2024-08-22 支援用 RAG 技術實現從自然語言到 SQL 語句的轉換。
 
 ## 🎉 關注項目
@@ -151,7 +180,7 @@
 > 所有 Docker 映像檔都是為 x86 平台建置的。目前，我們不提供 ARM64 平台的 Docker 映像檔。
 > 如果您使用的是 ARM64 平台，請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
 
-   > 執行以下指令會自動下載 RAGFlow slim Docker 映像 `v0.19.0-slim`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.19.0-slim` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。例如，你可以透過設定 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0` 來下載 RAGFlow 鏡像的 `v0.19.0` 完整發行版。
+   > 執行以下指令會自動下載 RAGFlow slim Docker 映像 `v0.19.1-slim`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.19.1-slim` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。例如，你可以透過設定 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` 來下載 RAGFlow 鏡像的 `v0.19.1` 完整發行版。
 
    ```bash
    $ cd ragflow/docker
@@ -164,8 +193,8 @@
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
    | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 

README_zh.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DBEDFA"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,6 +30,9 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
@@ -40,6 +43,31 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
+
+<details open>
+<summary><b>📕 目录</b></summary>
+
+- 💡 [RAGFlow 是什么？](#-RAGFlow-是什么)
+- 🎮 [Demo](#-demo)
+- 📌 [近期更新](#-近期更新)
+- 🌟 [主要功能](#-主要功能)
+- 🔎 [系统架构](#-系统架构)
+- 🎬 [快速开始](#-快速开始)
+- 🔧 [系统配置](#-系统配置)
+- 🔨 [以源代码启动服务](#-以源代码启动服务)
+- 📚 [技术文档](#-技术文档)
+- 📜 [路线图](#-路线图)
+- 🏄 [贡献指南](#-贡献指南)
+- 🙌 [加入社区](#-加入社区)
+- 🤝 [商务合作](#-商务合作)
+
+</details>
+
 ## 💡 RAGFlow 是什么？
 
 [RAGFlow](https://ragflow.io/) 是一款基于深度文档理解构建的开源 RAG（Retrieval-Augmented Generation）引擎。RAGFlow 可以为各种规模的企业及个人提供一套精简的 RAG 工作流程，结合大语言模型（LLM）针对用户各类不同的复杂格式数据提供可靠的问答以及有理有据的引用。
@@ -55,11 +83,11 @@
 
 ## 🔥 近期更新
 
-- 2025-03-19 PDF和DOCX中的图支持用多模态大模型去解析得到描述.
+- 2025-05-23 Agent 新增 Python/JS 代码执行器组件。
+- 2025-05-05 支持跨语言查询。
+- 2025-03-19 PDF 和 DOCX 中的图支持用多模态大模型去解析得到描述.
 - 2025-02-28 结合互联网搜索（Tavily），对于任意大模型实现类似 Deep Research 的推理功能.
-- 2025-01-26 优化知识图谱的提取和应用，提供了多种配置选择。
 - 2024-12-18 升级了 DeepDoc 的文档布局分析模型。
-- 2024-11-01 对解析后的 chunk 加入关键词抽取和相关问题生成以提高召回的准确度。
 - 2024-08-22 支持用 RAG 技术实现从自然语言到 SQL 语句的转换。
 
 ## 🎉 关注项目
@@ -152,7 +180,7 @@
 > 请注意，目前官方提供的所有 Docker 镜像均基于 x86 架构构建，并不提供基于 ARM64 的 Docker 镜像。
 > 如果你的操作系统是 ARM64 架构，请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
 
-   > 运行以下命令会自动下载 RAGFlow slim Docker 镜像 `v0.19.0-slim`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.19.0-slim` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。比如，你可以通过设置 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0` 来下载 RAGFlow 镜像的 `v0.19.0` 完整发行版。
+   > 运行以下命令会自动下载 RAGFlow slim Docker 镜像 `v0.19.1-slim`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.19.1-slim` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。比如，你可以通过设置 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` 来下载 RAGFlow 镜像的 `v0.19.1` 完整发行版。
 
    ```bash
    $ cd ragflow/docker
@@ -165,8 +193,8 @@
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
    | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 

agent/canvas.py

@@ -169,6 +169,7 @@ class Canvas:
     def run(self, running_hint_text = "is running...🕞", **kwargs):
         if not running_hint_text or not isinstance(running_hint_text, str):
             running_hint_text = "is running...🕞"
+        bypass_begin = bool(kwargs.get("bypass_begin", False))
 
         if self.answer:
             cpn_id = self.answer[0]
@@ -188,6 +189,12 @@ class Canvas:
         if not self.path:
             self.components["begin"]["obj"].run(self.history, **kwargs)
             self.path.append(["begin"])
+            if bypass_begin:
+                cpn = self.get_component("begin")
+                downstream = cpn["downstream"]
+                self.path.append(downstream)
+
+
 
         self.path.append([])
 
@@ -304,6 +311,8 @@ class Canvas:
 
     def get_history(self, window_size):
         convs = []
+        if window_size <= 0:
+            return convs
         for role, obj in self.history[window_size * -1:]:
             if isinstance(obj, list) and obj and all([isinstance(o, dict) for o in obj]):
                 convs.append({"role": role, "content": '\n'.join([str(s.get("content", "")) for s in obj])})

agent/component/answer.py

@@ -64,14 +64,17 @@ class Answer(ComponentBase, ABC):
             for ii, row in stream.iterrows():
                 answer += row.to_dict()["content"]
                 yield {"content": answer}
-        else:
+        elif stream is not None:
             for st in stream():
                 res = st
                 yield st
-        if self._param.post_answers:
+        if self._param.post_answers and res:
             res["content"] += random.choice(self._param.post_answers)
             yield res
 
+        if res is None:
+            res = {"content": ""}
+
         self.set_output(res)
 
     def set_exception(self, e):

agent/component/base.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-from abc import ABC
 import builtins
 import json
-import os
 import logging
+import os
+from abc import ABC
 from functools import partial
 from typing import Any, Tuple, Union
 
@@ -110,15 +110,11 @@ class ComponentParamBase(ABC):
         update_from_raw_conf = conf.get(_IS_RAW_CONF, True)
         if update_from_raw_conf:
             deprecated_params_set = self._get_or_init_deprecated_params_set()
-            feeded_deprecated_params_set = (
-                self._get_or_init_feeded_deprecated_params_set()
-            )
+            feeded_deprecated_params_set = self._get_or_init_feeded_deprecated_params_set()
             user_feeded_params_set = self._get_or_init_user_feeded_params_set()
             setattr(self, _IS_RAW_CONF, False)
         else:
-            feeded_deprecated_params_set = (
-                self._get_or_init_feeded_deprecated_params_set(conf)
-            )
+            feeded_deprecated_params_set = self._get_or_init_feeded_deprecated_params_set(conf)
             user_feeded_params_set = self._get_or_init_user_feeded_params_set(conf)
 
         def _recursive_update_param(param, config, depth, prefix):
@@ -154,15 +150,11 @@ class ComponentParamBase(ABC):
 
                 else:
                     # recursive set obj attr
-                    sub_params = _recursive_update_param(
-                        attr, config_value, depth + 1, prefix=f"{prefix}{config_key}."
-                    )
+                    sub_params = _recursive_update_param(attr, config_value, depth + 1, prefix=f"{prefix}{config_key}.")
                     setattr(param, config_key, sub_params)
 
             if not allow_redundant and redundant_attrs:
-                raise ValueError(
-                    f"cpn `{getattr(self, '_name', type(self))}` has redundant parameters: `{[redundant_attrs]}`"
-                )
+                raise ValueError(f"cpn `{getattr(self, '_name', type(self))}` has redundant parameters: `{[redundant_attrs]}`")
 
             return param
 
@@ -193,9 +185,7 @@ class ComponentParamBase(ABC):
         param_validation_path_prefix = home_dir + "/param_validation/"
 
         param_name = type(self).__name__
-        param_validation_path = "/".join(
-            [param_validation_path_prefix, param_name + ".json"]
-        )
+        param_validation_path = "/".join([param_validation_path_prefix, param_name + ".json"])
 
         validation_json = None
 
@@ -228,11 +218,7 @@ class ComponentParamBase(ABC):
                         break
 
                 if not value_legal:
-                    raise ValueError(
-                        "Plase check runtime conf, {} = {} does not match user-parameter restriction".format(
-                            variable, value
-                        )
-                    )
+                    raise ValueError("Plase check runtime conf, {} = {} does not match user-parameter restriction".format(variable, value))
 
             elif variable in validation_json:
                 self._validate_param(attr, validation_json)
@@ -240,94 +226,63 @@ class ComponentParamBase(ABC):
     @staticmethod
     def check_string(param, descr):
         if type(param).__name__ not in ["str"]:
-            raise ValueError(
-                descr + " {} not supported, should be string type".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be string type".format(param))
 
     @staticmethod
     def check_empty(param, descr):
         if not param:
-            raise ValueError(
-                descr + " does not support empty value."
-            )
+            raise ValueError(descr + " does not support empty value.")
 
     @staticmethod
     def check_positive_integer(param, descr):
         if type(param).__name__ not in ["int", "long"] or param <= 0:
-            raise ValueError(
-                descr + " {} not supported, should be positive integer".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be positive integer".format(param))
 
     @staticmethod
     def check_positive_number(param, descr):
         if type(param).__name__ not in ["float", "int", "long"] or param <= 0:
-            raise ValueError(
-                descr + " {} not supported, should be positive numeric".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be positive numeric".format(param))
 
     @staticmethod
     def check_nonnegative_number(param, descr):
         if type(param).__name__ not in ["float", "int", "long"] or param < 0:
-            raise ValueError(
-                descr
-                + " {} not supported, should be non-negative numeric".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be non-negative numeric".format(param))
 
     @staticmethod
     def check_decimal_float(param, descr):
         if type(param).__name__ not in ["float", "int"] or param < 0 or param > 1:
-            raise ValueError(
-                descr
-                + " {} not supported, should be a float number in range [0, 1]".format(
-                    param
-                )
-            )
+            raise ValueError(descr + " {} not supported, should be a float number in range [0, 1]".format(param))
 
     @staticmethod
     def check_boolean(param, descr):
         if type(param).__name__ != "bool":
-            raise ValueError(
-                descr + " {} not supported, should be bool type".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be bool type".format(param))
 
     @staticmethod
     def check_open_unit_interval(param, descr):
         if type(param).__name__ not in ["float"] or param <= 0 or param >= 1:
-            raise ValueError(
-                descr + " should be a numeric number between 0 and 1 exclusively"
-            )
+            raise ValueError(descr + " should be a numeric number between 0 and 1 exclusively")
 
     @staticmethod
     def check_valid_value(param, descr, valid_values):
         if param not in valid_values:
-            raise ValueError(
-                descr
-                + " {} is not supported, it should be in {}".format(param, valid_values)
-            )
+            raise ValueError(descr + " {} is not supported, it should be in {}".format(param, valid_values))
 
     @staticmethod
     def check_defined_type(param, descr, types):
         if type(param).__name__ not in types:
-            raise ValueError(
-                descr + " {} not supported, should be one of {}".format(param, types)
-            )
+            raise ValueError(descr + " {} not supported, should be one of {}".format(param, types))
 
     @staticmethod
     def check_and_change_lower(param, valid_list, descr=""):
         if type(param).__name__ != "str":
-            raise ValueError(
-                descr
-                + " {} not supported, should be one of {}".format(param, valid_list)
-            )
+            raise ValueError(descr + " {} not supported, should be one of {}".format(param, valid_list))
 
         lower_param = param.lower()
         if lower_param in valid_list:
             return lower_param
         else:
-            raise ValueError(
-                descr
-                + " {} not supported, should be one of {}".format(param, valid_list)
-            )
+            raise ValueError(descr + " {} not supported, should be one of {}".format(param, valid_list))
 
     @staticmethod
     def _greater_equal_than(value, limit):
@@ -341,11 +296,7 @@ class ComponentParamBase(ABC):
     def _range(value, ranges):
         in_range = False
         for left_limit, right_limit in ranges:
-            if (
-                    left_limit - settings.FLOAT_ZERO
-                    <= value
-                    <= right_limit + settings.FLOAT_ZERO
-            ):
+            if left_limit - settings.FLOAT_ZERO <= value <= right_limit + settings.FLOAT_ZERO:
                 in_range = True
                 break
 
@@ -361,16 +312,11 @@ class ComponentParamBase(ABC):
 
     def _warn_deprecated_param(self, param_name, descr):
         if self._deprecated_params_set.get(param_name):
-            logging.warning(
-                f"{descr} {param_name} is deprecated and ignored in this version."
-            )
+            logging.warning(f"{descr} {param_name} is deprecated and ignored in this version.")
 
     def _warn_to_deprecate_param(self, param_name, descr, new_param):
         if self._deprecated_params_set.get(param_name):
-            logging.warning(
-                f"{descr} {param_name} will be deprecated in future release; "
-                f"please use {new_param} instead."
-            )
+            logging.warning(f"{descr} {param_name} will be deprecated in future release; please use {new_param} instead.")
             return True
         return False
 
@@ -395,14 +341,16 @@ class ComponentBase(ABC):
             "params": {},
             "output": {},
             "inputs": {}
-        }}""".format(self.component_name,
-                     self._param,
-                     json.dumps(json.loads(str(self._param)).get("output", {}), ensure_ascii=False),
-                     json.dumps(json.loads(str(self._param)).get("inputs", []), ensure_ascii=False)
+        }}""".format(
+            self.component_name,
+            self._param,
+            json.dumps(json.loads(str(self._param)).get("output", {}), ensure_ascii=False),
+            json.dumps(json.loads(str(self._param)).get("inputs", []), ensure_ascii=False),
         )
 
     def __init__(self, canvas, id, param: ComponentParamBase):
         from agent.canvas import Canvas  # Local import to avoid cyclic dependency
+
         assert isinstance(canvas, Canvas), "canvas must be an instance of Canvas"
         self._canvas = canvas
         self._id = id
@@ -410,15 +358,17 @@ class ComponentBase(ABC):
         self._param.check()
 
     def get_dependent_components(self):
-        cpnts = set([para["component_id"].split("@")[0] for para in self._param.query \
-                     if para.get("component_id") \
-                     and para["component_id"].lower().find("answer") < 0 \
-                     and para["component_id"].lower().find("begin") < 0])
+        cpnts = set(
+            [
+                para["component_id"].split("@")[0]
+                for para in self._param.query
+                if para.get("component_id") and para["component_id"].lower().find("answer") < 0 and para["component_id"].lower().find("begin") < 0
+            ]
+        )
         return list(cpnts)
 
     def run(self, history, **kwargs):
-        logging.debug("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False),
-                                                              json.dumps(kwargs, ensure_ascii=False)))
+        logging.debug("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False), json.dumps(kwargs, ensure_ascii=False)))
         self._param.debug_inputs = []
         try:
             res = self._run(history, **kwargs)
@@ -465,7 +415,7 @@ class ComponentBase(ABC):
 
     def set_infor(self, v):
         setattr(self._param, self._param.infor_var_name, v)
-        
+
     def _fetch_outputs_from(self, sources: list[dict[str, Any]]) -> list[pd.DataFrame]:
         outs = []
         for q in sources:
@@ -482,7 +432,7 @@ class ComponentBase(ABC):
 
                 if q["component_id"].lower().find("answer") == 0:
                     txt = []
-                    for r, c in self._canvas.history[::-1][:self._param.message_history_window_size][::-1]:
+                    for r, c in self._canvas.history[::-1][: self._param.message_history_window_size][::-1]:
                         txt.append(f"{r.upper()}:{c}")
                     txt = "\n".join(txt)
                     outs.append(pd.DataFrame([{"content": txt}]))
@@ -512,21 +462,16 @@ class ComponentBase(ABC):
                 content: str
 
                 if len(records) > 1:
-                    content = "\n".join(
-                        [str(d["content"]) for d in records]
-                    )
+                    content = "\n".join([str(d["content"]) for d in records])
                 else:
                     content = records[0]["content"]
 
-                self._param.inputs.append({
-                    "component_id": records[0].get("component_id"),
-                    "content": content
-                })
+                self._param.inputs.append({"component_id": records[0].get("component_id"), "content": content})
 
             if outs:
                 df = pd.concat(outs, ignore_index=True)
                 if "content" in df:
-                    df = df.drop_duplicates(subset=['content']).reset_index(drop=True)
+                    df = df.drop_duplicates(subset=["content"]).reset_index(drop=True)
                 return df
 
         upstream_outs = []
@@ -540,9 +485,8 @@ class ComponentBase(ABC):
                     o["component_id"] = u
                     upstream_outs.append(o)
                     continue
-            #if self.component_name.lower()!="answer" and u not in self._canvas.get_component(self._id)["upstream"]: continue
-            if self.component_name.lower().find("switch") < 0 \
-                    and self.get_component_name(u) in ["relevant", "categorize"]:
+            # if self.component_name.lower()!="answer" and u not in self._canvas.get_component(self._id)["upstream"]: continue
+            if self.component_name.lower().find("switch") < 0 and self.get_component_name(u) in ["relevant", "categorize"]:
                 continue
             if u.lower().find("answer") >= 0:
                 for r, c in self._canvas.history[::-1]:
@@ -562,7 +506,7 @@ class ComponentBase(ABC):
 
         df = pd.concat(upstream_outs, ignore_index=True)
         if "content" in df:
-            df = df.drop_duplicates(subset=['content']).reset_index(drop=True)
+            df = df.drop_duplicates(subset=["content"]).reset_index(drop=True)
 
         self._param.inputs = []
         for _, r in df.iterrows():
@@ -614,5 +558,5 @@ class ComponentBase(ABC):
         return self._canvas.get_component(pid)["obj"]
 
     def get_upstream(self):
-        cpn_nms = self._canvas.get_component(self._id)['upstream']
+        cpn_nms = self._canvas.get_component(self._id)["upstream"]
         return cpn_nms

agent/component/categorize.py

@@ -99,9 +99,13 @@ class Categorize(Generate, ABC):
         # If a category is found, return the category with the highest count.
         if any(category_counts.values()):
             max_category = max(category_counts.items(), key=lambda x: x[1])
-            return Categorize.be_output(self._param.category_description[max_category[0]]["to"])
+            res = Categorize.be_output(self._param.category_description[max_category[0]]["to"])
+            self.set_output(res)
+            return res
 
-        return Categorize.be_output(list(self._param.category_description.items())[-1][1]["to"])
+        res = Categorize.be_output(list(self._param.category_description.items())[-1][1]["to"])
+        self.set_output(res)
+        return res
 
     def debug(self, **kwargs):
         df = self._run([], **kwargs)

agent/component/code.py

@@ -81,21 +81,32 @@ class Code(ComponentBase, ABC):
         for input in self._param.arguments:
             if "@" in input["component_id"]:
                 component_id = input["component_id"].split("@")[0]
-                refered_component_key = input["component_id"].split("@")[1]
-                refered_component = self._canvas.get_component(component_id)["obj"]
+                referred_component_key = input["component_id"].split("@")[1]
+                referred_component = self._canvas.get_component(component_id)["obj"]
 
-                for param in refered_component._param.query:
-                    if param["key"] == refered_component_key:
+                for param in referred_component._param.query:
+                    if param["key"] == referred_component_key:
                         if "value" in param:
                             arguments[input["name"]] = param["value"]
             else:
-                cpn = self._canvas.get_component(input["component_id"])["obj"]
-                if cpn.component_name.lower() == "answer":
-                    arguments[input["name"]] = self._canvas.get_history(1)[0]["content"]
-                    continue
-                _, out = cpn.output(allow_partial=False)
-                if not out.empty:
-                    arguments[input["name"]] = "\n".join(out["content"])
+                referred_component = self._canvas.get_component(input["component_id"])["obj"]
+                referred_component_name = referred_component.component_name
+                referred_component_id = referred_component._id
+
+                debug_inputs = self._param.debug_inputs
+                if debug_inputs:
+                    for param in debug_inputs:
+                        if param["key"] == referred_component_id:
+                            if "value" in param and param["name"] == input["name"]:
+                                arguments[input["name"]] = param["value"]
+                else:
+                    if referred_component_name.lower() == "answer":
+                        arguments[input["name"]] = self._canvas.get_history(1)[0]["content"]
+                        continue
+
+                    _, out = referred_component.output(allow_partial=False)
+                    if not out.empty:
+                        arguments[input["name"]] = "\n".join(out["content"])
 
         return self._execute_code(
             language=self._param.lang,
@@ -136,3 +147,6 @@ class Code(ComponentBase, ABC):
             cpn_id = input["component_id"]
             elements.append({"key": cpn_id, "name": input["name"]})
         return elements
+
+    def debug(self, **kwargs):
+        return self._run([], **kwargs)

agent/component/message.py

@@ -40,7 +40,9 @@ class Message(ComponentBase, ABC):
         if kwargs.get("stream"):
             return partial(self.stream_output)
 
-        return Message.be_output(random.choice(self._param.messages))
+        res = Message.be_output(random.choice(self._param.messages))
+        self.set_output(res)
+        return res
 
     def stream_output(self):
         res = None

agent/component/retrieval.py

@@ -96,6 +96,7 @@ class Retrieval(ComponentBase, ABC):
             rerank_mdl = LLMBundle(kbs[0].tenant_id, LLMType.RERANK, self._param.rerank_id)
 
         if kbs:
+            query = re.sub(r"^user[:：\s]*", "", query, flags=re.IGNORECASE)
             kbinfos = settings.retrievaler.retrieval(
                 query,
                 embd_mdl,

agent/component/template.py

@@ -15,8 +15,11 @@
 #
 import json
 import re
+
+from jinja2 import StrictUndefined
+from jinja2.sandbox import SandboxedEnvironment
+
 from agent.component.base import ComponentBase, ComponentParamBase
-from jinja2 import Template as Jinja2Template
 
 
 class TemplateParam(ComponentParamBase):
@@ -75,6 +78,11 @@ class Template(ComponentBase):
                     if p["key"] == key:
                         value = p.get("value", "")
                         self.make_kwargs(para, kwargs, value)
+
+                        origin_pattern = "{begin@" + key + "}"
+                        new_pattern = "begin_" + key
+                        content = content.replace(origin_pattern, new_pattern)
+                        kwargs[new_pattern] = kwargs.pop(origin_pattern, "")
                         break
                 else:
                     assert False, f"Can't find parameter '{key}' for {cpn_id}"
@@ -89,19 +97,27 @@ class Template(ComponentBase):
                 else:
                     hist = ""
                 self.make_kwargs(para, kwargs, hist)
+
+                if ":" in component_id:
+                    origin_pattern = "{" + component_id + "}"
+                    new_pattern = component_id.replace(":", "_")
+                    content = content.replace(origin_pattern, new_pattern)
+                    kwargs[new_pattern] = kwargs.pop(component_id, "")
                 continue
 
             _, out = cpn.output(allow_partial=False)
 
             result = ""
             if "content" in out.columns:
-                result = "\n".join(
-                    [o if isinstance(o, str) else str(o) for o in out["content"]]
-                )
+                result = "\n".join([o if isinstance(o, str) else str(o) for o in out["content"]])
 
             self.make_kwargs(para, kwargs, result)
 
-        template = Jinja2Template(content)
+        env = SandboxedEnvironment(
+            autoescape=True,
+            undefined=StrictUndefined,
+        )
+        template = env.from_string(content)
 
         try:
             content = template.render(kwargs)
@@ -114,19 +130,16 @@ class Template(ComponentBase):
                     v = json.dumps(v, ensure_ascii=False)
                 except Exception:
                     pass
-            content = re.sub(
-                r"\{%s\}" % re.escape(n), v, content
-            )
-            content = re.sub(
-                r"(#+)", r" \1 ", content
-            )
+            # Process backslashes in strings, Use Lambda function to avoid escape issues
+            if isinstance(v, str):
+                v = v.replace("\\", "\\\\")
+            content = re.sub(r"\{%s\}" % re.escape(n), lambda match: v, content)
+            content = re.sub(r"(#+)", r" \1 ", content)
 
         return Template.be_output(content)
 
     def make_kwargs(self, para, kwargs, value):
-        self._param.inputs.append(
-            {"component_id": para["key"], "content": value}
-        )
+        self._param.inputs.append({"component_id": para["key"], "content": value})
         try:
             value = json.loads(value)
         except Exception:

agent/templates/customer_service.json

@@ -52,7 +52,10 @@
             "parameters": [],
             "presence_penalty": 0.4,
             "prompt": "",
-            "query": [],
+            "query": [{
+              "type": "reference",
+              "component_id": "RewriteQuestion:AllNightsSniff"
+             }],
             "temperature": 0.1,
             "top_p": 0.3
           }
@@ -195,11 +198,15 @@
             "message_history_window_size": 22,
             "output": null,
             "output_var_name": "output",
-            "query": [],
             "rerank_id": "",
             "similarity_threshold": 0.2,
             "top_k": 1024,
-            "top_n": 6
+            "top_n": 6,
+            "query": [{
+              "type": "reference",
+              "component_id": "RewriteQuestion:AllNightsSniff"
+             }],
+            "use_kg": false
           }
         },
         "upstream": [
@@ -548,7 +555,11 @@
               "temperature": 0.1,
               "temperatureEnabled": true,
               "topPEnabled": true,
-              "top_p": 0.3
+              "top_p": 0.3,
+              "query": [{
+                "type": "reference",
+                "component_id": "RewriteQuestion:AllNightsSniff"
+              }]
             },
             "label": "Categorize",
             "name": "Question Categorize"
@@ -625,7 +636,11 @@
               "keywords_similarity_weight": 0.3,
               "similarity_threshold": 0.2,
               "top_k": 1024,
-              "top_n": 6
+              "top_n": 6,
+              "query": [{
+                "type": "reference",
+                "component_id": "RewriteQuestion:AllNightsSniff"
+              }]
             },
             "label": "Retrieval",
             "name": "Search product info"
@@ -932,7 +947,7 @@
             "y": 962.5655101584402
           },
           "resizing": false,
-          "selected": true,
+          "selected": false,
           "sourcePosition": "right",
           "style": {
             "height": 163,

agentic_reasoning/deep_research.py

@@ -36,17 +36,20 @@ class DeepResearcher:
         self._kb_retrieve = kb_retrieve
         self._kg_retrieve = kg_retrieve
 
-    @staticmethod
-    def _remove_query_tags(text):
-        """Remove query tags from text"""
-        pattern = re.escape(BEGIN_SEARCH_QUERY) + r"(.*?)" + re.escape(END_SEARCH_QUERY)
+    def _remove_tags(text: str, start_tag: str, end_tag: str) -> str:
+        """General Tag Removal Method"""
+        pattern = re.escape(start_tag) + r"(.*?)" + re.escape(end_tag)
         return re.sub(pattern, "", text)
 
     @staticmethod
-    def _remove_result_tags(text):
-        """Remove result tags from text"""
-        pattern = re.escape(BEGIN_SEARCH_RESULT) + r"(.*?)" + re.escape(END_SEARCH_RESULT)
-        return re.sub(pattern, "", text)
+    def _remove_query_tags(text: str) -> str:
+        """Remove Query Tags"""
+        return DeepResearcher._remove_tags(text, BEGIN_SEARCH_QUERY, END_SEARCH_QUERY)
+
+    @staticmethod
+    def _remove_result_tags(text: str) -> str:
+        """Remove Result Tags"""
+        return DeepResearcher._remove_tags(text, BEGIN_SEARCH_RESULT, END_SEARCH_RESULT)
 
     def _generate_reasoning(self, msg_history):
         """Generate reasoning steps"""
@@ -95,21 +98,31 @@ class DeepResearcher:
     def _retrieve_information(self, search_query):
         """Retrieve information from different sources"""
         # 1. Knowledge base retrieval
-        kbinfos = self._kb_retrieve(question=search_query) if self._kb_retrieve else {"chunks": [], "doc_aggs": []}
-        
+        kbinfos = []
+        try:
+            kbinfos = self._kb_retrieve(question=search_query) if self._kb_retrieve else {"chunks": [], "doc_aggs": []}
+        except Exception as e:
+            logging.error(f"Knowledge base retrieval error: {e}")
+
         # 2. Web retrieval (if Tavily API is configured)
-        if self.prompt_config.get("tavily_api_key"):
-            tav = Tavily(self.prompt_config["tavily_api_key"])
-            tav_res = tav.retrieve_chunks(search_query)
-            kbinfos["chunks"].extend(tav_res["chunks"])
-            kbinfos["doc_aggs"].extend(tav_res["doc_aggs"])
-        
+        try:
+            if self.prompt_config.get("tavily_api_key"):
+                tav = Tavily(self.prompt_config["tavily_api_key"])
+                tav_res = tav.retrieve_chunks(search_query)
+                kbinfos["chunks"].extend(tav_res["chunks"])
+                kbinfos["doc_aggs"].extend(tav_res["doc_aggs"])
+        except Exception as e:
+            logging.error(f"Web retrieval error: {e}")
+
         # 3. Knowledge graph retrieval (if configured)
-        if self.prompt_config.get("use_kg") and self._kg_retrieve:
-            ck = self._kg_retrieve(question=search_query)
-            if ck["content_with_weight"]:
-                kbinfos["chunks"].insert(0, ck)
-                
+        try:
+            if self.prompt_config.get("use_kg") and self._kg_retrieve:
+                ck = self._kg_retrieve(question=search_query)
+                if ck["content_with_weight"]:
+                    kbinfos["chunks"].insert(0, ck)
+        except Exception as e:
+            logging.error(f"Knowledge graph retrieval error: {e}")
+
         return kbinfos
 
     def _update_chunk_info(self, chunk_info, kbinfos):

api/apps/__init__.py

@@ -146,10 +146,23 @@ def load_user(web_request):
     if authorization:
         try:
             access_token = str(jwt.loads(authorization))
+            
+            if not access_token or not access_token.strip():
+                logging.warning("Authentication attempt with empty access token")
+                return None
+            
+            # Access tokens should be UUIDs (32 hex characters)
+            if len(access_token.strip()) < 32:
+                logging.warning(f"Authentication attempt with invalid token format: {len(access_token)} chars")
+                return None
+            
             user = UserService.query(
                 access_token=access_token, status=StatusEnum.VALID.value
             )
             if user:
+                if not user[0].access_token or not user[0].access_token.strip():
+                    logging.warning(f"User {user[0].email} has empty access_token in database")
+                    return None
                 return user[0]
             else:
                 return None

api/apps/api_app.py

@@ -18,7 +18,7 @@ import os
 import re
 from datetime import datetime, timedelta
 from flask import request, Response
-from api.db.services.llm_service import TenantLLMService
+from api.db.services.llm_service import LLMBundle
 from flask_login import login_required, current_user
 
 from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileType, LLMType, ParserType, FileSource
@@ -875,14 +875,12 @@ def retrieval():
                 data=False, message='Knowledge bases use different embedding models or does not exist."',
                 code=settings.RetCode.AUTHENTICATION_ERROR)
 
-        embd_mdl = TenantLLMService.model_instance(
-            kbs[0].tenant_id, LLMType.EMBEDDING.value, llm_name=kbs[0].embd_id)
+        embd_mdl = LLMBundle(kbs[0].tenant_id, LLMType.EMBEDDING, llm_name=kbs[0].embd_id)
         rerank_mdl = None
         if req.get("rerank_id"):
-            rerank_mdl = TenantLLMService.model_instance(
-                kbs[0].tenant_id, LLMType.RERANK.value, llm_name=req["rerank_id"])
+            rerank_mdl = LLMBundle(kbs[0].tenant_id, LLMType.RERANK, llm_name=req["rerank_id"])
         if req.get("keyword", False):
-            chat_mdl = TenantLLMService.model_instance(kbs[0].tenant_id, LLMType.CHAT)
+            chat_mdl = LLMBundle(kbs[0].tenant_id, LLMType.CHAT)
             question += keyword_extraction(chat_mdl, question)
         ranks = settings.retrievaler.retrieval(question, embd_mdl, kbs[0].tenant_id, kb_ids, page, size,
                                                similarity_threshold, vector_similarity_weight, top,

api/apps/auth/oidc.py

@@ -68,8 +68,7 @@ class OIDCClient(OAuthClient):
             alg = headers.get("alg", "RS256")
 
             # Use PyJWT's PyJWKClient to fetch JWKS and find signing key
-            jwks_url = f"{self.issuer}/.well-known/jwks.json"
-            jwks_cli = jwt.PyJWKClient(jwks_url)
+            jwks_cli = jwt.PyJWKClient(self.jwks_uri)
             signing_key = jwks_cli.get_signing_key_from_jwt(id_token).key
 
             # Decode and verify signature

api/apps/canvas_app.py

@@ -249,7 +249,9 @@ def debug():
                 code=RetCode.OPERATING_ERROR)
 
         canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id)
-        canvas.get_component(req["component_id"])["obj"]._param.debug_inputs = req["params"]
+        componant = canvas.get_component(req["component_id"])["obj"]
+        componant.reset()
+        componant._param.debug_inputs = req["params"]
         df = canvas.get_component(req["component_id"])["obj"].debug()
         return get_json_result(data=df.to_dict(orient="records"))
     except Exception as e:

api/apps/conversation_app.py

@@ -42,6 +42,7 @@ def set_conversation():
     conv_id = req.get("conversation_id")
     is_new = req.get("is_new")
     name = req.get("name", "New conversation")
+    req["user_id"] = current_user.id
 
     if len(name) > 255:
         name = name[0:255]
@@ -64,7 +65,7 @@ def set_conversation():
         e, dia = DialogService.get_by_id(req["dialog_id"])
         if not e:
             return get_data_error_result(message="Dialog not found")
-        conv = {"id": conv_id, "dialog_id": req["dialog_id"], "name": name, "message": [{"role": "assistant", "content": dia.prompt_config["prologue"]}]}
+        conv = {"id": conv_id, "dialog_id": req["dialog_id"], "name": name, "message": [{"role": "assistant", "content": dia.prompt_config["prologue"]}],"user_id": current_user.id}
         ConversationService.save(**conv)
         return get_json_result(data=conv)
     except Exception as e:
@@ -248,7 +249,7 @@ def completion():
         else:
             answer = None
             for ans in chat(dia, msg, **req):
-                answer = structure_answer(conv, ans, message_id, req["conversation_id"])
+                answer = structure_answer(conv, ans, message_id, conv.id)
                 ConversationService.update_by_id(conv.id, conv.to_dict())
                 break
             return get_json_result(data=answer)

api/apps/dialog_app.py

@@ -28,6 +28,7 @@ from api.utils.api_utils import get_json_result
 
 
 @manager.route('/set', methods=['POST'])  # noqa: F821
+@validate_request("prompt_config")
 @login_required
 def set_dialog():
     req = request.json
@@ -43,33 +44,10 @@ def set_dialog():
     similarity_threshold = req.get("similarity_threshold", 0.1)
     vector_similarity_weight = req.get("vector_similarity_weight", 0.3)
     llm_setting = req.get("llm_setting", {})
-    default_prompt_with_dataset = {
-        "system": """你是一个智能助手，请总结知识库的内容来回答问题，请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时，你的回答必须包括“知识库中未找到您要的答案！”这句话。回答需要考虑聊天历史。
-以下是知识库：
-{knowledge}
-以上是知识库。""",
-        "prologue": "您好，我是您的助手小樱，长得可爱又善良，can I help you?",
-        "parameters": [
-            {"key": "knowledge", "optional": False}
-        ],
-        "empty_response": "Sorry! 知识库中未找到相关内容！"
-    }
-    default_prompt_no_dataset = {
-        "system": """You are a helpful assistant.""",
-        "prologue": "您好，我是您的助手小樱，长得可爱又善良，can I help you?",
-        "parameters": [
-           
-        ],
-        "empty_response": ""
-    }
-    prompt_config = req.get("prompt_config", default_prompt_with_dataset)
-
-    if not prompt_config["system"]:
-        prompt_config["system"] = default_prompt_with_dataset["system"]
+    prompt_config = req["prompt_config"]
     
-    if not req.get("kb_ids", []):
-        if prompt_config['system'] == default_prompt_with_dataset['system'] or "{knowledge}" in prompt_config['system']:
-            prompt_config = default_prompt_no_dataset
+    if not req.get("kb_ids", []) and not prompt_config.get("tavily_api_key") and "{knowledge}" in prompt_config['system']:
+        return get_data_error_result(message="Please remove `{knowledge}` in system prompt since no knowledge base/Tavily used here.")
 
     for p in prompt_config["parameters"]:
         if p["optional"]:

api/apps/document_app.py

@@ -23,7 +23,7 @@ from flask import request
 from flask_login import current_user, login_required
 
 from api import settings
-from api.constants import IMG_BASE64_PREFIX
+from api.constants import FILE_NAME_LEN_LIMIT, IMG_BASE64_PREFIX
 from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileSource, FileType, ParserType, TaskStatus
 from api.db.db_models import File, Task
 from api.db.services import duplicate_name
@@ -61,18 +61,21 @@ def upload():
     for file_obj in file_objs:
         if file_obj.filename == "":
             return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
+        if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
 
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
     err, files = FileService.upload_document(kb, file_objs, current_user.id)
 
+    if err:
+        return get_json_result(data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
+
     if not files:
         return get_json_result(data=files, message="There seems to be an issue with your file format. Please verify it is correct and not corrupted.", code=settings.RetCode.DATA_ERROR)
     files = [f[0] for f in files]  # remove the blob
 
-    if err:
-        return get_json_result(data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
     return get_json_result(data=files)
 
 
@@ -146,6 +149,12 @@ def create():
     kb_id = req["kb_id"]
     if not kb_id:
         return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+    if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+        return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+
+    if req["name"].strip() == "":
+        return get_json_result(data=False, message="File name can't be empty.", code=settings.RetCode.ARGUMENT_ERROR)
+    req["name"] = req["name"].strip()
 
     try:
         e, kb = KnowledgebaseService.get_by_id(kb_id)
@@ -190,7 +199,10 @@ def list_docs():
     page_number = int(request.args.get("page", 0))
     items_per_page = int(request.args.get("page_size", 0))
     orderby = request.args.get("orderby", "create_time")
-    desc = request.args.get("desc", True)
+    if request.args.get("desc", "true").lower() == "false":
+        desc = False
+    else:
+        desc = True
 
     req = request.get_json()
 
@@ -401,6 +413,9 @@ def rename():
             return get_data_error_result(message="Document not found!")
         if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
             return get_json_result(data=False, message="The extension of file can't be changed", code=settings.RetCode.ARGUMENT_ERROR)
+        if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+
         for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
             if d.name == req["name"]:
                 return get_data_error_result(message="Duplicated document name in the same knowledgebase.")

api/apps/kb_app.py

@@ -34,6 +34,7 @@ from api import settings
 from rag.nlp import search
 from api.constants import DATASET_NAME_LIMIT
 from rag.settings import PAGERANK_FLD
+from rag.utils.storage_factory import STORAGE_IMPL
 
 
 @manager.route('/create', methods=['post'])  # noqa: F821
@@ -44,11 +45,11 @@ def create():
     dataset_name = req["name"]
     if not isinstance(dataset_name, str):
         return get_data_error_result(message="Dataset name must be string.")
-    if dataset_name == "":
+    if dataset_name.strip() == "":
         return get_data_error_result(message="Dataset name can't be empty.")
-    if len(dataset_name) >= DATASET_NAME_LIMIT:
+    if len(dataset_name.encode("utf-8")) > DATASET_NAME_LIMIT:
         return get_data_error_result(
-            message=f"Dataset name length is {len(dataset_name)} which is large than {DATASET_NAME_LIMIT}")
+            message=f"Dataset name length is {len(dataset_name)} which is larger than {DATASET_NAME_LIMIT}")
 
     dataset_name = dataset_name.strip()
     dataset_name = duplicate_name(
@@ -78,7 +79,15 @@ def create():
 @not_allowed_parameters("id", "tenant_id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by")
 def update():
     req = request.json
+    if not isinstance(req["name"], str):
+        return get_data_error_result(message="Dataset name must be string.")
+    if req["name"].strip() == "":
+        return get_data_error_result(message="Dataset name can't be empty.")
+    if len(req["name"].encode("utf-8")) > DATASET_NAME_LIMIT:
+        return get_data_error_result(
+            message=f"Dataset name length is {len(req['name'])} which is large than {DATASET_NAME_LIMIT}")
     req["name"] = req["name"].strip()
+
     if not KnowledgebaseService.accessible4deletion(req["kb_id"], current_user.id):
         return get_json_result(
             data=False,
@@ -106,7 +115,7 @@ def update():
 
         if req["name"].lower() != kb.name.lower() \
                 and len(
-            KnowledgebaseService.query(name=req["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) > 1:
+            KnowledgebaseService.query(name=req["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) >= 1:
             return get_data_error_result(
                 message="Duplicated knowledgebase name.")
 
@@ -115,6 +124,9 @@ def update():
             return get_data_error_result()
 
         if kb.pagerank != req.get("pagerank", 0):
+            if os.environ.get("DOC_ENGINE", "elasticsearch") != "elasticsearch":
+                return get_data_error_result(message="'pagerank' can only be set when doc_engine is elasticsearch")
+            
             if req.get("pagerank", 0) > 0:
                 settings.docStoreConn.update({"kb_id": kb.id}, {PAGERANK_FLD: req["pagerank"]},
                                          search.index_name(kb.tenant_id), kb.id)
@@ -167,7 +179,10 @@ def list_kbs():
     items_per_page = int(request.args.get("page_size", 0))
     parser_id = request.args.get("parser_id")
     orderby = request.args.get("orderby", "create_time")
-    desc = request.args.get("desc", True)
+    if request.args.get("desc", "true").lower() == "false":
+        desc = False
+    else:
+        desc = True
 
     req = request.get_json()
     owner_ids = req.get("owner_ids", [])
@@ -184,9 +199,9 @@ def list_kbs():
                 tenants, current_user.id, 0,
                 0, orderby, desc, keywords, parser_id)
             kbs = [kb for kb in kbs if kb["tenant_id"] in tenants]
+            total = len(kbs)
             if page_number and items_per_page:
                 kbs = kbs[(page_number-1)*items_per_page:page_number*items_per_page]
-            total = len(kbs)
         return get_json_result(data={"kbs": kbs, "total": total})
     except Exception as e:
         return server_error_response(e)
@@ -226,6 +241,8 @@ def rm():
         for kb in kbs:
             settings.docStoreConn.delete({"kb_id": kb.id}, search.index_name(kb.tenant_id), kb.id)
             settings.docStoreConn.deleteIdx(search.index_name(kb.tenant_id), kb.id)
+            if hasattr(STORAGE_IMPL, 'remove_bucket'):
+                STORAGE_IMPL.remove_bucket(kb.id)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)

api/apps/sdk/chat.py

@@ -173,8 +173,10 @@ def update(tenant_id, chat_id):
     if llm:
         if "model_name" in llm:
             req["llm_id"] = llm.pop("model_name")
-            if not TenantLLMService.query(tenant_id=tenant_id, llm_name=req["llm_id"], model_type="chat"):
-                return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
+            if req.get("llm_id") is not None:
+                llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(req["llm_id"])
+                if not TenantLLMService.query(tenant_id=tenant_id, llm_name=llm_name, llm_factory=llm_factory, model_type="chat"):
+                    return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
         req["llm_setting"] = req.pop("llm")
     e, tenant = TenantService.get_by_id(tenant_id)
     if not e:

api/apps/sdk/dataset.py

@@ -16,10 +16,12 @@
 
 
 import logging
+import os
 
 from flask import request
 from peewee import OperationalError
 
+from api import settings
 from api.db import FileSource, StatusEnum
 from api.db.db_models import File
 from api.db.services.document_service import DocumentService
@@ -48,6 +50,8 @@ from api.utils.validation_utils import (
     validate_and_parse_json_request,
     validate_and_parse_request_args,
 )
+from rag.nlp import search
+from rag.settings import PAGERANK_FLD
 
 
 @manager.route("/datasets", methods=["POST"])  # noqa: F821
@@ -97,9 +101,6 @@ def create(tenant_id):
                      "picture", "presentation", "qa", "table", "tag"
                      ]
               description: Chunking method.
-            pagerank:
-              type: integer
-              description: Set page rank.
             parser_config:
               type: object
               description: Parser configuration.
@@ -124,48 +125,36 @@ def create(tenant_id):
     try:
         if KnowledgebaseService.get_or_none(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
             return get_error_operating_result(message=f"Dataset name '{req['name']}' already exists")
-    except OperationalError as e:
-        logging.exception(e)
-        return get_error_data_result(message="Database operation failed")
 
-    req["parser_config"] = get_parser_config(req["parser_id"], req["parser_config"])
-    req["id"] = get_uuid()
-    req["tenant_id"] = tenant_id
-    req["created_by"] = tenant_id
+        req["parser_config"] = get_parser_config(req["parser_id"], req["parser_config"])
+        req["id"] = get_uuid()
+        req["tenant_id"] = tenant_id
+        req["created_by"] = tenant_id
 
-    try:
         ok, t = TenantService.get_by_id(tenant_id)
         if not ok:
             return get_error_permission_result(message="Tenant not found")
-    except OperationalError as e:
-        logging.exception(e)
-        return get_error_data_result(message="Database operation failed")
 
-    if not req.get("embd_id"):
-        req["embd_id"] = t.embd_id
-    else:
-        ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
-        if not ok:
-            return err
+        if not req.get("embd_id"):
+            req["embd_id"] = t.embd_id
+        else:
+            ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
+            if not ok:
+                return err
 
-    try:
         if not KnowledgebaseService.save(**req):
             return get_error_data_result(message="Create dataset error.(Database error)")
-    except OperationalError as e:
-        logging.exception(e)
-        return get_error_data_result(message="Database operation failed")
 
-    try:
         ok, k = KnowledgebaseService.get_by_id(req["id"])
         if not ok:
             return get_error_data_result(message="Dataset created failed")
+
+        response_data = remap_dictionary_keys(k.to_dict())
+        return get_result(data=response_data)
     except OperationalError as e:
         logging.exception(e)
         return get_error_data_result(message="Database operation failed")
 
-    response_data = remap_dictionary_keys(k.to_dict())
-    return get_result(data=response_data)
-
 
 @manager.route("/datasets", methods=["DELETE"])  # noqa: F821
 @token_required
@@ -211,34 +200,27 @@ def delete(tenant_id):
     if err is not None:
         return get_error_argument_result(err)
 
-    kb_id_instance_pairs = []
-    if req["ids"] is None:
-        try:
+    try:
+        kb_id_instance_pairs = []
+        if req["ids"] is None:
             kbs = KnowledgebaseService.query(tenant_id=tenant_id)
             for kb in kbs:
                 kb_id_instance_pairs.append((kb.id, kb))
-        except OperationalError as e:
-            logging.exception(e)
-            return get_error_data_result(message="Database operation failed")
-    else:
-        error_kb_ids = []
-        for kb_id in req["ids"]:
-            try:
+
+        else:
+            error_kb_ids = []
+            for kb_id in req["ids"]:
                 kb = KnowledgebaseService.get_or_none(id=kb_id, tenant_id=tenant_id)
                 if kb is None:
                     error_kb_ids.append(kb_id)
                     continue
                 kb_id_instance_pairs.append((kb_id, kb))
-            except OperationalError as e:
-                logging.exception(e)
-                return get_error_data_result(message="Database operation failed")
-        if len(error_kb_ids) > 0:
-            return get_error_permission_result(message=f"""User '{tenant_id}' lacks permission for datasets: '{", ".join(error_kb_ids)}'""")
+            if len(error_kb_ids) > 0:
+                return get_error_permission_result(message=f"""User '{tenant_id}' lacks permission for datasets: '{", ".join(error_kb_ids)}'""")
 
-    errors = []
-    success_count = 0
-    for kb_id, kb in kb_id_instance_pairs:
-        try:
+        errors = []
+        success_count = 0
+        for kb_id, kb in kb_id_instance_pairs:
             for doc in DocumentService.query(kb_id=kb_id):
                 if not DocumentService.remove_document(doc, tenant_id):
                     errors.append(f"Remove document '{doc.id}' error for dataset '{kb_id}'")
@@ -256,18 +238,18 @@ def delete(tenant_id):
                 errors.append(f"Delete dataset error for {kb_id}")
                 continue
             success_count += 1
-        except OperationalError as e:
-            logging.exception(e)
-            return get_error_data_result(message="Database operation failed")
 
-    if not errors:
-        return get_result()
+        if not errors:
+            return get_result()
 
-    error_message = f"Successfully deleted {success_count} datasets, {len(errors)} failed. Details: {'; '.join(errors)[:128]}..."
-    if success_count == 0:
-        return get_error_data_result(message=error_message)
+        error_message = f"Successfully deleted {success_count} datasets, {len(errors)} failed. Details: {'; '.join(errors)[:128]}..."
+        if success_count == 0:
+            return get_error_data_result(message=error_message)
 
-    return get_result(data={"success_count": success_count, "errors": errors[:5]}, message=error_message)
+        return get_result(data={"success_count": success_count, "errors": errors[:5]}, message=error_message)
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
 
 
 @manager.route("/datasets/<dataset_id>", methods=["PUT"])  # noqa: F821
@@ -349,44 +331,51 @@ def update(tenant_id, dataset_id):
         kb = KnowledgebaseService.get_or_none(id=dataset_id, tenant_id=tenant_id)
         if kb is None:
             return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{dataset_id}'")
-    except OperationalError as e:
-        logging.exception(e)
-        return get_error_data_result(message="Database operation failed")
 
-    if req.get("parser_config"):
-        req["parser_config"] = deep_merge(kb.parser_config, req["parser_config"])
+        if req.get("parser_config"):
+            req["parser_config"] = deep_merge(kb.parser_config, req["parser_config"])
 
-    if (chunk_method := req.get("parser_id")) and chunk_method != kb.parser_id:
-        if not req.get("parser_config"):
-            req["parser_config"] = get_parser_config(chunk_method, None)
-    elif "parser_config" in req and not req["parser_config"]:
-        del req["parser_config"]
+        if (chunk_method := req.get("parser_id")) and chunk_method != kb.parser_id:
+            if not req.get("parser_config"):
+                req["parser_config"] = get_parser_config(chunk_method, None)
+        elif "parser_config" in req and not req["parser_config"]:
+            del req["parser_config"]
 
-    if "name" in req and req["name"].lower() != kb.name.lower():
-        try:
+        if "name" in req and req["name"].lower() != kb.name.lower():
             exists = KnowledgebaseService.get_or_none(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)
             if exists:
                 return get_error_data_result(message=f"Dataset name '{req['name']}' already exists")
-        except OperationalError as e:
-            logging.exception(e)
-            return get_error_data_result(message="Database operation failed")
 
-    if "embd_id" in req:
-        if kb.chunk_num != 0 and req["embd_id"] != kb.embd_id:
-            return get_error_data_result(message=f"When chunk_num ({kb.chunk_num}) > 0, embedding_model must remain {kb.embd_id}")
-        ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
-        if not ok:
-            return err
+        if "embd_id" in req:
+            if kb.chunk_num != 0 and req["embd_id"] != kb.embd_id:
+                return get_error_data_result(message=f"When chunk_num ({kb.chunk_num}) > 0, embedding_model must remain {kb.embd_id}")
+            ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
+            if not ok:
+                return err
+
+        if "pagerank" in req and req["pagerank"] != kb.pagerank:
+            if os.environ.get("DOC_ENGINE", "elasticsearch") == "infinity":
+                return get_error_argument_result(message="'pagerank' can only be set when doc_engine is elasticsearch")
+
+            if req["pagerank"] > 0:
+                settings.docStoreConn.update({"kb_id": kb.id}, {PAGERANK_FLD: req["pagerank"]}, search.index_name(kb.tenant_id), kb.id)
+            else:
+                # Elasticsearch requires PAGERANK_FLD be non-zero!
+                settings.docStoreConn.update({"exists": PAGERANK_FLD}, {"remove": PAGERANK_FLD}, search.index_name(kb.tenant_id), kb.id)
 
-    try:
         if not KnowledgebaseService.update_by_id(kb.id, req):
             return get_error_data_result(message="Update dataset error.(Database error)")
+
+        ok, k = KnowledgebaseService.get_by_id(kb.id)
+        if not ok:
+            return get_error_data_result(message="Dataset created failed")
+
+        response_data = remap_dictionary_keys(k.to_dict())
+        return get_result(data=response_data)
     except OperationalError as e:
         logging.exception(e)
         return get_error_data_result(message="Database operation failed")
 
-    return get_result()
-
 
 @manager.route("/datasets", methods=["GET"])  # noqa: F821
 @token_required
@@ -450,26 +439,19 @@ def list_datasets(tenant_id):
     if err is not None:
         return get_error_argument_result(err)
 
-    kb_id = request.args.get("id")
-    name = args.get("name")
-    if kb_id:
-        try:
-            kbs = KnowledgebaseService.get_kb_by_id(kb_id, tenant_id)
-        except OperationalError as e:
-            logging.exception(e)
-            return get_error_data_result(message="Database operation failed")
-        if not kbs:
-            return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{kb_id}'")
-    if name:
-        try:
-            kbs = KnowledgebaseService.get_kb_by_name(name, tenant_id)
-        except OperationalError as e:
-            logging.exception(e)
-            return get_error_data_result(message="Database operation failed")
-        if not kbs:
-            return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{name}'")
-
     try:
+        kb_id = request.args.get("id")
+        name = args.get("name")
+        if kb_id:
+            kbs = KnowledgebaseService.get_kb_by_id(kb_id, tenant_id)
+
+            if not kbs:
+                return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{kb_id}'")
+        if name:
+            kbs = KnowledgebaseService.get_kb_by_name(name, tenant_id)
+            if not kbs:
+                return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{name}'")
+
         tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)
         kbs = KnowledgebaseService.get_list(
             [m["tenant_id"] for m in tenants],
@@ -481,11 +463,11 @@ def list_datasets(tenant_id):
             kb_id,
             name,
         )
+
+        response_data_list = []
+        for kb in kbs:
+            response_data_list.append(remap_dictionary_keys(kb))
+        return get_result(data=response_data_list)
     except OperationalError as e:
         logging.exception(e)
         return get_error_data_result(message="Database operation failed")
-
-    response_data_list = []
-    for kb in kbs:
-        response_data_list.append(remap_dictionary_keys(kb))
-    return get_result(data=response_data_list)

api/apps/sdk/dify_retrieval.py

@@ -16,6 +16,7 @@
 from flask import request, jsonify
 
 from api.db import LLMType
+from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api import settings
@@ -70,12 +71,13 @@ def retrieval(tenant_id):
 
         records = []
         for c in ranks["chunks"]:
+            e, doc = DocumentService.get_by_id( c["doc_id"])
             c.pop("vector", None)
             records.append({
                 "content": c["content_with_weight"],
                 "score": c["similarity"],
                 "title": c["docnm_kwd"],
-                "metadata": {}
+                "metadata": doc.meta_fields
             })
 
         return jsonify({"records": records})

api/apps/sdk/doc.py

@@ -13,38 +13,35 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import pathlib
 import datetime
-
-from rag.app.qa import rmPrefix, beAdoc
-from rag.nlp import rag_tokenizer
-from api.db import LLMType, ParserType
-from api.db.services.llm_service import TenantLLMService, LLMBundle
-from api import settings
-import xxhash
+import logging
+import pathlib
 import re
-from api.utils.api_utils import token_required
-from api.db.db_models import Task
-from api.db.services.task_service import TaskService, queue_tasks
-from api.utils.api_utils import server_error_response
-from api.utils.api_utils import get_result, get_error_data_result
 from io import BytesIO
+
+import xxhash
 from flask import request, send_file
-from api.db import FileSource, TaskStatus, FileType
-from api.db.db_models import File
+from peewee import OperationalError
+from pydantic import BaseModel, Field, validator
+
+from api import settings
+from api.constants import FILE_NAME_LEN_LIMIT
+from api.db import FileSource, FileType, LLMType, ParserType, TaskStatus
+from api.db.db_models import File, Task
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.utils.api_utils import construct_json_result, get_parser_config, check_duplicate_ids
-from rag.nlp import search
-from rag.prompts import keyword_extraction
+from api.db.services.llm_service import LLMBundle, TenantLLMService
+from api.db.services.task_service import TaskService, queue_tasks
+from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_parser_config, get_result, server_error_response, token_required
+from rag.app.qa import beAdoc, rmPrefix
 from rag.app.tag import label_question
+from rag.nlp import rag_tokenizer, search
+from rag.prompts import keyword_extraction
 from rag.utils import rmSpace
 from rag.utils.storage_factory import STORAGE_IMPL
 
-from pydantic import BaseModel, Field, validator
-
 MAXIMUM_OF_UPLOADING_FILES = 256
 
 
@@ -60,7 +57,7 @@ class Chunk(BaseModel):
     available: bool = True
     positions: list[list[int]] = Field(default_factory=list)
 
-    @validator('positions')
+    @validator("positions")
     def validate_positions(cls, value):
         for sublist in value:
             if len(sublist) != 5:
@@ -128,20 +125,14 @@ def upload(dataset_id, tenant_id):
                     description: Processing status.
     """
     if "file" not in request.files:
-        return get_error_data_result(
-            message="No file part!", code=settings.RetCode.ARGUMENT_ERROR
-        )
+        return get_error_data_result(message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
     file_objs = request.files.getlist("file")
     for file_obj in file_objs:
         if file_obj.filename == "":
-            return get_result(
-                message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR
-            )
-        if len(file_obj.filename.encode("utf-8")) >= 128:
-            return get_result(
-                message="File name should be less than 128 bytes.", code=settings.RetCode.ARGUMENT_ERROR
-            )
-    '''
+            return get_result(message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
+        if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+            return get_result(message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+    """
     # total size
     total_size = 0
     for file_obj in file_objs:
@@ -154,7 +145,7 @@ def upload(dataset_id, tenant_id):
             message=f"Total file size exceeds 10MB limit! ({total_size / (1024 * 1024):.2f} MB)",
             code=settings.RetCode.ARGUMENT_ERROR,
         )
-    '''
+    """
     e, kb = KnowledgebaseService.get_by_id(dataset_id)
     if not e:
         raise LookupError(f"Can't find the dataset with ID {dataset_id}!")
@@ -236,8 +227,7 @@ def update_doc(tenant_id, dataset_id, document_id):
         return get_error_data_result(message="You don't own the dataset.")
     e, kb = KnowledgebaseService.get_by_id(dataset_id)
     if not e:
-        return get_error_data_result(
-            message="Can't find this knowledgebase!")
+        return get_error_data_result(message="Can't find this knowledgebase!")
     doc = DocumentService.query(kb_id=dataset_id, id=document_id)
     if not doc:
         return get_error_data_result(message="The dataset doesn't own the document.")
@@ -258,24 +248,19 @@ def update_doc(tenant_id, dataset_id, document_id):
         DocumentService.update_meta_fields(document_id, req["meta_fields"])
 
     if "name" in req and req["name"] != doc.name:
-        if len(req["name"].encode("utf-8")) >= 128:
+        if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
             return get_result(
-                message="The name should be less than 128 bytes.",
+                message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.",
                 code=settings.RetCode.ARGUMENT_ERROR,
             )
-        if (
-                pathlib.Path(req["name"].lower()).suffix
-                != pathlib.Path(doc.name.lower()).suffix
-        ):
+        if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
             return get_result(
                 message="The extension of file can't be changed",
                 code=settings.RetCode.ARGUMENT_ERROR,
             )
         for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
             if d.name == req["name"]:
-                return get_error_data_result(
-                    message="Duplicated document name in the same dataset."
-                )
+                return get_error_data_result(message="Duplicated document name in the same dataset.")
         if not DocumentService.update_by_id(document_id, {"name": req["name"]}):
             return get_error_data_result(message="Database error (Document rename)!")
 
@@ -287,46 +272,28 @@ def update_doc(tenant_id, dataset_id, document_id):
     if "parser_config" in req:
         DocumentService.update_parser_config(doc.id, req["parser_config"])
     if "chunk_method" in req:
-        valid_chunk_method = {
-            "naive",
-            "manual",
-            "qa",
-            "table",
-            "paper",
-            "book",
-            "laws",
-            "presentation",
-            "picture",
-            "one",
-            "knowledge_graph",
-            "email",
-            "tag"
-        }
+        valid_chunk_method = {"naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one", "knowledge_graph", "email", "tag"}
         if req.get("chunk_method") not in valid_chunk_method:
-            return get_error_data_result(
-                f"`chunk_method` {req['chunk_method']} doesn't exist"
-            )
-        if doc.parser_id.lower() == req["chunk_method"].lower():
-            return get_result()
+            return get_error_data_result(f"`chunk_method` {req['chunk_method']} doesn't exist")
 
         if doc.type == FileType.VISUAL or re.search(r"\.(ppt|pptx|pages)$", doc.name):
             return get_error_data_result(message="Not supported yet!")
 
-        e = DocumentService.update_by_id(
-            doc.id,
-            {
-                "parser_id": req["chunk_method"],
-                "progress": 0,
-                "progress_msg": "",
-                "run": TaskStatus.UNSTART.value,
-            },
-        )
-        if not e:
-            return get_error_data_result(message="Document not found!")
-        req["parser_config"] = get_parser_config(
-            req["chunk_method"], req.get("parser_config")
-        )
-        DocumentService.update_parser_config(doc.id, req["parser_config"])
+        if doc.parser_id.lower() != req["chunk_method"].lower():
+            e = DocumentService.update_by_id(
+                doc.id,
+                {
+                    "parser_id": req["chunk_method"],
+                    "progress": 0,
+                    "progress_msg": "",
+                    "run": TaskStatus.UNSTART.value,
+                },
+            )
+            if not e:
+                return get_error_data_result(message="Document not found!")
+        if not req.get("parser_config"):
+            req["parser_config"] = get_parser_config(req["chunk_method"], req.get("parser_config"))
+            DocumentService.update_parser_config(doc.id, req["parser_config"])
         if doc.token_num > 0:
             e = DocumentService.increment_chunk_num(
                 doc.id,
@@ -343,19 +310,45 @@ def update_doc(tenant_id, dataset_id, document_id):
         status = int(req["enabled"])
         if doc.status != req["enabled"]:
             try:
-                if not DocumentService.update_by_id(
-                        doc.id, {"status": str(status)}):
-                    return get_error_data_result(
-                        message="Database error (Document update)!")
+                if not DocumentService.update_by_id(doc.id, {"status": str(status)}):
+                    return get_error_data_result(message="Database error (Document update)!")
 
-                settings.docStoreConn.update({"doc_id": doc.id}, {"available_int": status},
-                                             search.index_name(kb.tenant_id), doc.kb_id)
+                settings.docStoreConn.update({"doc_id": doc.id}, {"available_int": status}, search.index_name(kb.tenant_id), doc.kb_id)
                 return get_result(data=True)
             except Exception as e:
                 return server_error_response(e)
 
-    return get_result()
+    try:
+        ok, doc = DocumentService.get_by_id(doc.id)
+        if not ok:
+            return get_error_data_result(message="Dataset created failed")
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
 
+    key_mapping = {
+        "chunk_num": "chunk_count",
+        "kb_id": "dataset_id",
+        "token_num": "token_count",
+        "parser_id": "chunk_method",
+    }
+    run_mapping = {
+        "0": "UNSTART",
+        "1": "RUNNING",
+        "2": "CANCEL",
+        "3": "DONE",
+        "4": "FAIL",
+    }
+    renamed_doc = {}
+    for key, value in doc.to_dict().items():
+        if key == "run":
+            renamed_doc["run"] = run_mapping.get(str(value))
+        new_key = key_mapping.get(key, key)
+        renamed_doc[new_key] = value
+        if key == "run":
+            renamed_doc["run"] = run_mapping.get(value)
+
+    return get_result(data=renamed_doc)
 
 
 @manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["GET"])  # noqa: F821
@@ -397,25 +390,17 @@ def download(tenant_id, dataset_id, document_id):
           type: object
     """
     if not document_id:
-        return get_error_data_result(
-            message="Specify document_id please."
-        )
+        return get_error_data_result(message="Specify document_id please.")
     if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
         return get_error_data_result(message=f"You do not own the dataset {dataset_id}.")
     doc = DocumentService.query(kb_id=dataset_id, id=document_id)
     if not doc:
-        return get_error_data_result(
-            message=f"The dataset not own the document {document_id}."
-        )
+        return get_error_data_result(message=f"The dataset not own the document {document_id}.")
     # The process of downloading
-    doc_id, doc_location = File2DocumentService.get_storage_address(
-        doc_id=document_id
-    )  # minio address
+    doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id)  # minio address
     file_stream = STORAGE_IMPL.get(doc_id, doc_location)
     if not file_stream:
-        return construct_json_result(
-            message="This file is empty.", code=settings.RetCode.DATA_ERROR
-        )
+        return construct_json_result(message="This file is empty.", code=settings.RetCode.DATA_ERROR)
     file = BytesIO(file_stream)
     # Use send_file with a proper filename and MIME type
     return send_file(
@@ -530,9 +515,7 @@ def list_docs(dataset_id, tenant_id):
         desc = False
     else:
         desc = True
-    docs, tol = DocumentService.get_list(
-        dataset_id, page, page_size, orderby, desc, keywords, id, name
-    )
+    docs, tol = DocumentService.get_list(dataset_id, page, page_size, orderby, desc, keywords, id, name)
 
     # rename key's name
     renamed_doc_list = []
@@ -638,9 +621,7 @@ def delete(tenant_id, dataset_id):
             b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
 
             if not DocumentService.remove_document(doc, tenant_id):
-                return get_error_data_result(
-                    message="Database error (Document removal)!"
-                )
+                return get_error_data_result(message="Database error (Document removal)!")
 
             f2d = File2DocumentService.get_by_document_id(doc_id)
             FileService.filter_delete(
@@ -664,7 +645,10 @@ def delete(tenant_id, dataset_id):
 
     if duplicate_messages:
         if success_count > 0:
-            return get_result(message=f"Partially deleted {success_count} datasets with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages},)
+            return get_result(
+                message=f"Partially deleted {success_count} datasets with {len(duplicate_messages)} errors",
+                data={"success_count": success_count, "errors": duplicate_messages},
+            )
         else:
             return get_error_data_result(message=";".join(duplicate_messages))
 
@@ -729,9 +713,7 @@ def parse(tenant_id, dataset_id):
         if not doc:
             return get_error_data_result(message=f"You don't own the document {id}.")
         if 0.0 < doc[0].progress < 1.0:
-            return get_error_data_result(
-                "Can't parse document that is currently being processed"
-            )
+            return get_error_data_result("Can't parse document that is currently being processed")
         info = {"run": "1", "progress": 0, "progress_msg": "", "chunk_num": 0, "token_num": 0}
         DocumentService.update_by_id(id, info)
         settings.docStoreConn.delete({"doc_id": id}, search.index_name(tenant_id), dataset_id)
@@ -746,7 +728,10 @@ def parse(tenant_id, dataset_id):
         return get_result(message=f"Documents not found: {not_found}", code=settings.RetCode.DATA_ERROR)
     if duplicate_messages:
         if success_count > 0:
-            return get_result(message=f"Partially parsed {success_count} documents with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages},)
+            return get_result(
+                message=f"Partially parsed {success_count} documents with {len(duplicate_messages)} errors",
+                data={"success_count": success_count, "errors": duplicate_messages},
+            )
         else:
             return get_error_data_result(message=";".join(duplicate_messages))
 
@@ -808,16 +793,17 @@ def stop_parsing(tenant_id, dataset_id):
         if not doc:
             return get_error_data_result(message=f"You don't own the document {id}.")
         if int(doc[0].progress) == 1 or doc[0].progress == 0:
-            return get_error_data_result(
-                "Can't stop parsing document with progress at 0 or 1"
-            )
+            return get_error_data_result("Can't stop parsing document with progress at 0 or 1")
         info = {"run": "2", "progress": 0, "chunk_num": 0}
         DocumentService.update_by_id(id, info)
         settings.docStoreConn.delete({"doc_id": doc[0].id}, search.index_name(tenant_id), dataset_id)
         success_count += 1
     if duplicate_messages:
         if success_count > 0:
-            return get_result(message=f"Partially stopped {success_count} documents with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages},)
+            return get_result(
+                message=f"Partially stopped {success_count} documents with {len(duplicate_messages)} errors",
+                data={"success_count": success_count, "errors": duplicate_messages},
+            )
         else:
             return get_error_data_result(message=";".join(duplicate_messages))
     return get_result()
@@ -906,9 +892,7 @@ def list_chunks(tenant_id, dataset_id, document_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
     doc = DocumentService.query(id=document_id, kb_id=dataset_id)
     if not doc:
-        return get_error_data_result(
-            message=f"You don't own the document {document_id}."
-        )
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
     doc = doc[0]
     req = request.args
     doc_id = document_id
@@ -956,34 +940,29 @@ def list_chunks(tenant_id, dataset_id, document_id):
             del chunk[n]
         if not chunk:
             return get_error_data_result(f"Chunk `{req.get('id')}` not found.")
-        res['total'] = 1
+        res["total"] = 1
         final_chunk = {
-            "id":chunk.get("id",chunk.get("chunk_id")),
-            "content":chunk["content_with_weight"],
-            "document_id":chunk.get("doc_id",chunk.get("document_id")),
-            "docnm_kwd":chunk["docnm_kwd"],
-            "important_keywords":chunk.get("important_kwd",[]),
-            "questions":chunk.get("question_kwd",[]),
-            "dataset_id":chunk.get("kb_id",chunk.get("dataset_id")),
-            "image_id":chunk.get("img_id", ""),
-            "available":bool(chunk.get("available_int",1)),
-            "positions":chunk.get("position_int",[]),
+            "id": chunk.get("id", chunk.get("chunk_id")),
+            "content": chunk["content_with_weight"],
+            "document_id": chunk.get("doc_id", chunk.get("document_id")),
+            "docnm_kwd": chunk["docnm_kwd"],
+            "important_keywords": chunk.get("important_kwd", []),
+            "questions": chunk.get("question_kwd", []),
+            "dataset_id": chunk.get("kb_id", chunk.get("dataset_id")),
+            "image_id": chunk.get("img_id", ""),
+            "available": bool(chunk.get("available_int", 1)),
+            "positions": chunk.get("position_int", []),
         }
         res["chunks"].append(final_chunk)
         _ = Chunk(**final_chunk)
 
     elif settings.docStoreConn.indexExist(search.index_name(tenant_id), dataset_id):
-        sres = settings.retrievaler.search(query, search.index_name(tenant_id), [dataset_id], emb_mdl=None,
-                                           highlight=True)
+        sres = settings.retrievaler.search(query, search.index_name(tenant_id), [dataset_id], emb_mdl=None, highlight=True)
         res["total"] = sres.total
         for id in sres.ids:
             d = {
                 "id": id,
-                "content": (
-                    rmSpace(sres.highlight[id])
-                    if question and id in sres.highlight
-                    else sres.field[id].get("content_with_weight", "")
-                ),
+                "content": (rmSpace(sres.highlight[id]) if question and id in sres.highlight else sres.field[id].get("content_with_weight", "")),
                 "document_id": sres.field[id]["doc_id"],
                 "docnm_kwd": sres.field[id]["docnm_kwd"],
                 "important_keywords": sres.field[id].get("important_kwd", []),
@@ -991,10 +970,10 @@ def list_chunks(tenant_id, dataset_id, document_id):
                 "dataset_id": sres.field[id].get("kb_id", sres.field[id].get("dataset_id")),
                 "image_id": sres.field[id].get("img_id", ""),
                 "available": bool(int(sres.field[id].get("available_int", "1"))),
-                "positions": sres.field[id].get("position_int",[]),
+                "positions": sres.field[id].get("position_int", []),
             }
             res["chunks"].append(d)
-            _ = Chunk(**d) # validate the chunk
+            _ = Chunk(**d)  # validate the chunk
     return get_result(data=res)
 
 
@@ -1070,23 +1049,17 @@ def add_chunk(tenant_id, dataset_id, document_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
     doc = DocumentService.query(id=document_id, kb_id=dataset_id)
     if not doc:
-        return get_error_data_result(
-            message=f"You don't own the document {document_id}."
-        )
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
     doc = doc[0]
     req = request.json
     if not str(req.get("content", "")).strip():
         return get_error_data_result(message="`content` is required")
     if "important_keywords" in req:
         if not isinstance(req["important_keywords"], list):
-            return get_error_data_result(
-                "`important_keywords` is required to be a list"
-            )
+            return get_error_data_result("`important_keywords` is required to be a list")
     if "questions" in req:
         if not isinstance(req["questions"], list):
-            return get_error_data_result(
-                "`questions` is required to be a list"
-            )
+            return get_error_data_result("`questions` is required to be a list")
     chunk_id = xxhash.xxh64((req["content"] + document_id).encode("utf-8")).hexdigest()
     d = {
         "id": chunk_id,
@@ -1095,22 +1068,16 @@ def add_chunk(tenant_id, dataset_id, document_id):
     }
     d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
     d["important_kwd"] = req.get("important_keywords", [])
-    d["important_tks"] = rag_tokenizer.tokenize(
-        " ".join(req.get("important_keywords", []))
-    )
+    d["important_tks"] = rag_tokenizer.tokenize(" ".join(req.get("important_keywords", [])))
     d["question_kwd"] = [str(q).strip() for q in req.get("questions", []) if str(q).strip()]
-    d["question_tks"] = rag_tokenizer.tokenize(
-        "\n".join(req.get("questions", []))
-    )
+    d["question_tks"] = rag_tokenizer.tokenize("\n".join(req.get("questions", [])))
     d["create_time"] = str(datetime.datetime.now()).replace("T", " ")[:19]
     d["create_timestamp_flt"] = datetime.datetime.now().timestamp()
     d["kb_id"] = dataset_id
     d["docnm_kwd"] = doc.name
     d["doc_id"] = document_id
     embd_id = DocumentService.get_embd_id(document_id)
-    embd_mdl = TenantLLMService.model_instance(
-        tenant_id, LLMType.EMBEDDING.value, embd_id
-    )
+    embd_mdl = TenantLLMService.model_instance(tenant_id, LLMType.EMBEDDING.value, embd_id)
     v, c = embd_mdl.encode([doc.name, req["content"] if not d["question_kwd"] else "\n".join(d["question_kwd"])])
     v = 0.1 * v[0] + 0.9 * v[1]
     d["q_%d_vec" % len(v)] = v.tolist()
@@ -1203,7 +1170,10 @@ def rm_chunk(tenant_id, dataset_id, document_id):
             return get_result(message=f"deleted {chunk_number} chunks")
         return get_error_data_result(message=f"rm_chunk deleted chunks {chunk_number}, expect {len(unique_chunk_ids)}")
     if duplicate_messages:
-        return get_result(message=f"Partially deleted {chunk_number} chunks with {len(duplicate_messages)} errors", data={"success_count": chunk_number, "errors": duplicate_messages},)
+        return get_result(
+            message=f"Partially deleted {chunk_number} chunks with {len(duplicate_messages)} errors",
+            data={"success_count": chunk_number, "errors": duplicate_messages},
+        )
     return get_result(message=f"deleted {chunk_number} chunks")
 
 
@@ -1271,9 +1241,7 @@ def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
     doc = DocumentService.query(id=document_id, kb_id=dataset_id)
     if not doc:
-        return get_error_data_result(
-            message=f"You don't own the document {document_id}."
-        )
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
     doc = doc[0]
     req = request.json
     if "content" in req:
@@ -1296,19 +1264,13 @@ def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
     if "available" in req:
         d["available_int"] = int(req["available"])
     embd_id = DocumentService.get_embd_id(document_id)
-    embd_mdl = TenantLLMService.model_instance(
-        tenant_id, LLMType.EMBEDDING.value, embd_id
-    )
+    embd_mdl = TenantLLMService.model_instance(tenant_id, LLMType.EMBEDDING.value, embd_id)
     if doc.parser_id == ParserType.QA:
         arr = [t for t in re.split(r"[\n\t]", d["content_with_weight"]) if len(t) > 1]
         if len(arr) != 2:
-            return get_error_data_result(
-                message="Q&A must be separated by TAB/ENTER key."
-            )
+            return get_error_data_result(message="Q&A must be separated by TAB/ENTER key.")
         q, a = rmPrefix(arr[0]), rmPrefix(arr[1])
-        d = beAdoc(
-            d, arr[0], arr[1], not any([rag_tokenizer.is_chinese(t) for t in q + a])
-        )
+        d = beAdoc(d, arr[0], arr[1], not any([rag_tokenizer.is_chinese(t) for t in q + a]))
 
     v, c = embd_mdl.encode([doc.name, d["content_with_weight"] if not d.get("question_kwd") else "\n".join(d["question_kwd"])])
     v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
@@ -1425,9 +1387,7 @@ def retrieval_test(tenant_id):
     doc_ids_list = KnowledgebaseService.list_documents_by_ids(kb_ids)
     for doc_id in doc_ids:
         if doc_id not in doc_ids_list:
-            return get_error_data_result(
-                f"The datasets don't own the document {doc_id}"
-            )
+            return get_error_data_result(f"The datasets don't own the document {doc_id}")
     similarity_threshold = float(req.get("similarity_threshold", 0.2))
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
     top = int(req.get("top_k", 1024))
@@ -1463,14 +1423,10 @@ def retrieval_test(tenant_id):
             doc_ids,
             rerank_mdl=rerank_mdl,
             highlight=highlight,
-            rank_feature=label_question(question, kbs)
+            rank_feature=label_question(question, kbs),
         )
         if use_kg:
-            ck = settings.kg_retrievaler.retrieval(question,
-                                                   [k.tenant_id for k in kbs],
-                                                   kb_ids,
-                                                   embd_mdl,
-                                                   LLMBundle(kb.tenant_id, LLMType.CHAT))
+            ck = settings.kg_retrievaler.retrieval(question, [k.tenant_id for k in kbs], kb_ids, embd_mdl, LLMBundle(kb.tenant_id, LLMType.CHAT))
             if ck["content_with_weight"]:
                 ranks["chunks"].insert(0, ck)
 
@@ -1487,7 +1443,7 @@ def retrieval_test(tenant_id):
                 "important_kwd": "important_keywords",
                 "question_kwd": "questions",
                 "docnm_kwd": "document_keyword",
-                "kb_id":"dataset_id"
+                "kb_id": "dataset_id",
             }
             rename_chunk = {}
             for key, value in chunk.items():

api/apps/sdk/session.py

@@ -388,10 +388,10 @@ def agents_completion_openai_compatibility (tenant_id, agent_id):
     question = next((m["content"] for m in reversed(messages) if m["role"] == "user"), "")
     
     if req.get("stream", True):
-        return Response(completionOpenAI(tenant_id, agent_id, question, session_id=req.get("id", ""), stream=True), mimetype="text/event-stream")
+        return Response(completionOpenAI(tenant_id, agent_id, question, session_id=req.get("id", req.get("metadata", {}).get("id","")), stream=True), mimetype="text/event-stream")
     else:
         # For non-streaming, just return the response directly
-        response = next(completionOpenAI(tenant_id, agent_id, question, session_id=req.get("id", ""), stream=False))
+        response = next(completionOpenAI(tenant_id, agent_id, question, session_id=req.get("id", req.get("metadata", {}).get("id","")), stream=False))
         return jsonify(response)
     
 
@@ -464,7 +464,7 @@ def list_session(tenant_id, chat_id):
         if conv["reference"]:
             messages = conv["messages"]
             message_num = 0
-            while message_num < len(messages):
+            while message_num < len(messages) and message_num < len(conv["reference"]):
                 if message_num != 0 and messages[message_num]["role"] != "user":
                     chunk_list = []
                     if "chunks" in conv["reference"][message_num]:

api/apps/search_app.py

@@ -0,0 +1,188 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from flask import request
+from flask_login import current_user, login_required
+
+from api import settings
+from api.constants import DATASET_NAME_LIMIT
+from api.db import StatusEnum
+from api.db.db_models import DB
+from api.db.services import duplicate_name
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.search_service import SearchService
+from api.db.services.user_service import TenantService, UserTenantService
+from api.utils import get_uuid
+from api.utils.api_utils import get_data_error_result, get_json_result, not_allowed_parameters, server_error_response, validate_request
+
+
+@manager.route("/create", methods=["post"])  # noqa: F821
+@login_required
+@validate_request("name")
+def create():
+    req = request.get_json()
+    search_name = req["name"]
+    description = req.get("description", "")
+    if not isinstance(search_name, str):
+        return get_data_error_result(message="Search name must be string.")
+    if search_name.strip() == "":
+        return get_data_error_result(message="Search name can't be empty.")
+    if len(search_name.encode("utf-8")) > DATASET_NAME_LIMIT:
+        return get_data_error_result(message=f"Search name length is {len(search_name)} which is large than {DATASET_NAME_LIMIT}")
+    e, _ = TenantService.get_by_id(current_user.id)
+    if not e:
+        return get_data_error_result(message="Authorizationd identity.")
+
+    search_name = search_name.strip()
+    search_name = duplicate_name(KnowledgebaseService.query, name=search_name, tenant_id=current_user.id, status=StatusEnum.VALID.value)
+
+    req["id"] = get_uuid()
+    req["name"] = search_name
+    req["description"] = description
+    req["tenant_id"] = current_user.id
+    req["created_by"] = current_user.id
+    with DB.atomic():
+        try:
+            if not SearchService.save(**req):
+                return get_data_error_result()
+            return get_json_result(data={"search_id": req["id"]})
+        except Exception as e:
+            return server_error_response(e)
+
+
+@manager.route("/update", methods=["post"])  # noqa: F821
+@login_required
+@validate_request("search_id", "name", "search_config", "tenant_id")
+@not_allowed_parameters("id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by")
+def update():
+    req = request.get_json()
+    if not isinstance(req["name"], str):
+        return get_data_error_result(message="Search name must be string.")
+    if req["name"].strip() == "":
+        return get_data_error_result(message="Search name can't be empty.")
+    if len(req["name"].encode("utf-8")) > DATASET_NAME_LIMIT:
+        return get_data_error_result(message=f"Search name length is {len(req['name'])} which is large than {DATASET_NAME_LIMIT}")
+    req["name"] = req["name"].strip()
+    tenant_id = req["tenant_id"]
+    e, _ = TenantService.get_by_id(tenant_id)
+    if not e:
+        return get_data_error_result(message="Authorizationd identity.")
+
+    search_id = req["search_id"]
+    if not SearchService.accessible4deletion(search_id, current_user.id):
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+
+    try:
+        search_app = SearchService.query(tenant_id=tenant_id, id=search_id)[0]
+        if not search_app:
+            return get_json_result(data=False, message=f"Cannot find search {search_id}", code=settings.RetCode.DATA_ERROR)
+
+        if req["name"].lower() != search_app.name.lower() and len(SearchService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) >= 1:
+            return get_data_error_result(message="Duplicated search name.")
+
+        if "search_config" in req:
+            current_config = search_app.search_config or {}
+            new_config = req["search_config"]
+
+            if not isinstance(new_config, dict):
+                return get_data_error_result(message="search_config must be a JSON object")
+
+            updated_config = {**current_config, **new_config}
+            req["search_config"] = updated_config
+
+        req.pop("search_id", None)
+        req.pop("tenant_id", None)
+
+        updated = SearchService.update_by_id(search_id, req)
+        if not updated:
+            return get_data_error_result(message="Failed to update search")
+
+        e, updated_search = SearchService.get_by_id(search_id)
+        if not e:
+            return get_data_error_result(message="Failed to fetch updated search")
+
+        return get_json_result(data=updated_search.to_dict())
+
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/detail", methods=["GET"])  # noqa: F821
+@login_required
+def detail():
+    search_id = request.args["search_id"]
+    try:
+        tenants = UserTenantService.query(user_id=current_user.id)
+        for tenant in tenants:
+            if SearchService.query(tenant_id=tenant.tenant_id, id=search_id):
+                break
+        else:
+            return get_json_result(data=False, message="Has no permission for this operation.", code=settings.RetCode.OPERATING_ERROR)
+
+        search = SearchService.get_detail(search_id)
+        if not search:
+            return get_data_error_result(message="Can't find this Search App!")
+        return get_json_result(data=search)
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/list", methods=["POST"])  # noqa: F821
+@login_required
+def list_search_app():
+    keywords = request.args.get("keywords", "")
+    page_number = int(request.args.get("page", 0))
+    items_per_page = int(request.args.get("page_size", 0))
+    orderby = request.args.get("orderby", "create_time")
+    if request.args.get("desc", "true").lower() == "false":
+        desc = False
+    else:
+        desc = True
+
+    req = request.get_json()
+    owner_ids = req.get("owner_ids", [])
+    try:
+        if not owner_ids:
+            tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
+            tenants = [m["tenant_id"] for m in tenants]
+            search_apps, total = SearchService.get_by_tenant_ids(tenants, current_user.id, page_number, items_per_page, orderby, desc, keywords)
+        else:
+            tenants = owner_ids
+            search_apps, total = SearchService.get_by_tenant_ids(tenants, current_user.id, 0, 0, orderby, desc, keywords)
+            search_apps = [search_app for search_app in search_apps if search_app["tenant_id"] in tenants]
+            total = len(search_apps)
+            if page_number and items_per_page:
+                search_apps = search_apps[(page_number - 1) * items_per_page : page_number * items_per_page]
+        return get_json_result(data={"search_apps": search_apps, "total": total})
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/rm", methods=["post"])  # noqa: F821
+@login_required
+@validate_request("search_id")
+def rm():
+    req = request.get_json()
+    search_id = req["search_id"]
+    if not SearchService.accessible4deletion(search_id, current_user.id):
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+
+    try:
+        if not SearchService.delete_by_id(search_id):
+            return get_data_error_result(message=f"Failed to delete search App {search_id}")
+        return get_json_result(data=True)
+    except Exception as e:
+        return server_error_response(e)

api/apps/user_app.py

@@ -16,6 +16,7 @@
 import json
 import logging
 import re
+import secrets
 from datetime import datetime
 
 from flask import redirect, request, session
@@ -465,7 +466,7 @@ def log_out():
         schema:
           type: object
     """
-    current_user.access_token = ""
+    current_user.access_token = f"INVALID_{secrets.token_hex(16)}"
     current_user.save()
     logout_user()
     return get_json_result(data=True)

api/constants.py

@@ -13,9 +13,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-NAME_LENGTH_LIMIT = 2 ** 10
+NAME_LENGTH_LIMIT = 2**10
 
-IMG_BASE64_PREFIX = 'data:image/png;base64,'
+IMG_BASE64_PREFIX = "data:image/png;base64,"
 
 SERVICE_CONF = "service_conf.yaml"
 
@@ -25,3 +25,4 @@ REQUEST_WAIT_SEC = 2
 REQUEST_MAX_WAIT_SEC = 300
 
 DATASET_NAME_LIMIT = 128
+FILE_NAME_LEN_LIMIT = 255

api/db/db_models.py

@@ -13,16 +13,16 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import hashlib
 import inspect
 import logging
 import operator
 import os
 import sys
-import typing
 import time
+import typing
 from enum import Enum
 from functools import wraps
-import hashlib
 
 from flask_login import UserMixin
 from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
@@ -264,14 +264,15 @@ class BaseDataBase:
 
 def with_retry(max_retries=3, retry_delay=1.0):
     """Decorator: Add retry mechanism to database operations
-    
+
     Args:
         max_retries (int): maximum number of retries
         retry_delay (float): initial retry delay (seconds), will increase exponentially
-        
+
     Returns:
         decorated function
     """
+
     def decorator(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
@@ -284,26 +285,28 @@ def with_retry(max_retries=3, retry_delay=1.0):
                     # get self and method name for logging
                     self_obj = args[0] if args else None
                     func_name = func.__name__
-                    lock_name = getattr(self_obj, 'lock_name', 'unknown') if self_obj else 'unknown'
-                    
+                    lock_name = getattr(self_obj, "lock_name", "unknown") if self_obj else "unknown"
+
                     if retry < max_retries - 1:
-                        current_delay = retry_delay * (2 ** retry)
-                        logging.warning(f"{func_name} {lock_name} failed: {str(e)}, retrying ({retry+1}/{max_retries})")
+                        current_delay = retry_delay * (2**retry)
+                        logging.warning(f"{func_name} {lock_name} failed: {str(e)}, retrying ({retry + 1}/{max_retries})")
                         time.sleep(current_delay)
                     else:
                         logging.error(f"{func_name} {lock_name} failed after all attempts: {str(e)}")
-            
+
             if last_exception:
                 raise last_exception
             return False
+
         return wrapper
+
     return decorator
 
 
 class PostgresDatabaseLock:
     def __init__(self, lock_name, timeout=10, db=None):
         self.lock_name = lock_name
-        self.lock_id = int(hashlib.md5(lock_name.encode()).hexdigest(), 16) % (2**31-1)
+        self.lock_id = int(hashlib.md5(lock_name.encode()).hexdigest(), 16) % (2**31 - 1)
         self.timeout = int(timeout)
         self.db = db if db else DB
 
@@ -542,7 +545,7 @@ class LLM(DataBaseModel):
     max_tokens = IntegerField(default=0)
 
     tags = CharField(max_length=255, null=False, help_text="LLM, Text Embedding, Image2Text, Chat, 32k...", index=True)
-    is_tools =  BooleanField(null=False, help_text="support tools", default=False)
+    is_tools = BooleanField(null=False, help_text="support tools", default=False)
     status = CharField(max_length=1, null=True, help_text="is it validate(0: wasted, 1: validate)", default="1", index=True)
 
     def __str__(self):
@@ -796,6 +799,50 @@ class UserCanvasVersion(DataBaseModel):
         db_table = "user_canvas_version"
 
 
+class Search(DataBaseModel):
+    id = CharField(max_length=32, primary_key=True)
+    avatar = TextField(null=True, help_text="avatar base64 string")
+    tenant_id = CharField(max_length=32, null=False, index=True)
+    name = CharField(max_length=128, null=False, help_text="Search name", index=True)
+    description = TextField(null=True, help_text="KB description")
+    created_by = CharField(max_length=32, null=False, index=True)
+    search_config = JSONField(
+        null=False,
+        default={
+            "kb_ids": [],
+            "doc_ids": [],
+            "similarity_threshold": 0.0,
+            "vector_similarity_weight": 0.3,
+            "use_kg": False,
+            # rerank settings
+            "rerank_id": "",
+            "top_k": 1024,
+            # chat settings
+            "summary": False,
+            "chat_id": "",
+            "llm_setting": {
+                "temperature": 0.1,
+                "top_p": 0.3,
+                "frequency_penalty": 0.7,
+                "presence_penalty": 0.4,
+            },
+            "chat_settingcross_languages": [],
+            "highlight": False,
+            "keyword": False,
+            "web_search": False,
+            "related_search": False,
+            "query_mindmap": False,
+        },
+    )
+    status = CharField(max_length=1, null=True, help_text="is it validate(0: wasted, 1: validate)", default="1", index=True)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        db_table = "search"
+
+
 def migrate_db():
     migrator = DatabaseMigrator[settings.DATABASE_TYPE.upper()].value(DB)
     try:

api/db/init_data.py

@@ -84,14 +84,14 @@ def init_superuser():
         {"role": "user", "content": "Hello!"}], gen_conf={})
     if msg.find("ERROR: ") == 0:
         logging.error(
-            "'{}' dosen't work. {}".format(
+            "'{}' doesn't work. {}".format(
                 tenant["llm_id"],
                 msg))
     embd_mdl = LLMBundle(tenant["id"], LLMType.EMBEDDING, tenant["embd_id"])
     v, c = embd_mdl.encode(["Hello!"])
     if c == 0:
         logging.error(
-            "'{}' dosen't work!".format(
+            "'{}' doesn't work!".format(
                 tenant["embd_id"]))
 
 

api/db/services/__init__.py

@@ -13,27 +13,87 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import pathlib
 import re
+from pathlib import PurePath
+
 from .user_service import UserService as UserService
 
 
-def duplicate_name(query_func, **kwargs):
-    fnm = kwargs["name"]
-    objs = query_func(**kwargs)
-    if not objs:
-        return fnm
-    ext = pathlib.Path(fnm).suffix #.jpg
-    nm = re.sub(r"%s$"%ext, "", fnm)
-    r = re.search(r"\(([0-9]+)\)$", nm)
-    c = 0
-    if r:
-        c = int(r.group(1))
-        nm = re.sub(r"\([0-9]+\)$", "", nm)
-    c += 1
-    nm = f"{nm}({c})"
-    if ext:
-        nm += f"{ext}"
+def split_name_counter(filename: str) -> tuple[str, int | None]:
+    """
+    Splits a filename into main part and counter (if present in parentheses).
 
-    kwargs["name"] = nm
-    return duplicate_name(query_func, **kwargs)
+    Args:
+        filename: Input filename string to be parsed
+
+    Returns:
+        A tuple containing:
+        - The main filename part (string)
+        - The counter from parentheses (integer) or None if no counter exists
+    """
+    pattern = re.compile(r"^(.*?)\((\d+)\)$")
+
+    match = pattern.search(filename)
+    if match:
+        main_part = match.group(1).rstrip()
+        bracket_part = match.group(2)
+        return main_part, int(bracket_part)
+
+    return filename, None
+
+
+def duplicate_name(query_func, **kwargs) -> str:
+    """
+    Generates a unique filename by appending/incrementing a counter when duplicates exist.
+
+    Continuously checks for name availability using the provided query function,
+    automatically appending (1), (2), etc. until finding an available name or
+    reaching maximum retries.
+
+    Args:
+        query_func: Callable that accepts keyword arguments and returns:
+                  - True if name exists (should be modified)
+                  - False if name is available
+        **kwargs: Must contain 'name' key with original filename to check
+
+    Returns:
+        str: Available filename, either:
+            - Original name (if available)
+            - Modified name with counter (e.g., "file(1).txt")
+
+    Raises:
+        KeyError: If 'name' key not provided in kwargs
+        RuntimeError: If unable to generate unique name after maximum retries
+
+    Example:
+        >>> def name_exists(name): return name in existing_files
+        >>> duplicate_name(name_exists, name="document.pdf")
+        'document(1).pdf'  # If original exists
+    """
+    MAX_RETRIES = 1000
+
+    if "name" not in kwargs:
+        raise KeyError("Arguments must contain 'name' key")
+
+    original_name = kwargs["name"]
+    current_name = original_name
+    retries = 0
+
+    while retries < MAX_RETRIES:
+        if not query_func(**kwargs):
+            return current_name
+
+        path = PurePath(current_name)
+        stem = path.stem
+        suffix = path.suffix
+
+        main_part, counter = split_name_counter(stem)
+        counter = counter + 1 if counter else 1
+
+        new_name = f"{main_part}({counter}){suffix}"
+
+        kwargs["name"] = new_name
+        current_name = new_name
+        retries += 1
+
+    raise RuntimeError(f"Failed to generate unique name within {MAX_RETRIES} attempts. Original: {original_name}")

api/db/services/canvas_service.py

@@ -73,11 +73,11 @@ class UserCanvasService(CommonService):
                 User.nickname,
                 User.avatar.alias('tenant_avatar'),
             ]
-            angents = cls.model.select(*fields) \
+            agents = cls.model.select(*fields) \
             .join(User, on=(cls.model.user_id == User.id)) \
             .where(cls.model.id == pid)
             # obj = cls.model.query(id=pid)[0]
-            return True, angents.dicts()[0]
+            return True, agents.dicts()[0]
         except Exception as e:
             print(e)
             return False, None
@@ -100,25 +100,25 @@ class UserCanvasService(CommonService):
             cls.model.update_time
         ]
         if keywords:
-            angents = cls.model.select(*fields).join(User, on=(cls.model.user_id == User.id)).where(
+            agents = cls.model.select(*fields).join(User, on=(cls.model.user_id == User.id)).where(
                 ((cls.model.user_id.in_(joined_tenant_ids) & (cls.model.permission == 
                                                                 TenantPermission.TEAM.value)) | (
                     cls.model.user_id == user_id)),
                 (fn.LOWER(cls.model.title).contains(keywords.lower()))
             )
         else:
-            angents = cls.model.select(*fields).join(User, on=(cls.model.user_id == User.id)).where(
+            agents = cls.model.select(*fields).join(User, on=(cls.model.user_id == User.id)).where(
                 ((cls.model.user_id.in_(joined_tenant_ids) & (cls.model.permission == 
                                                                 TenantPermission.TEAM.value)) | (
                     cls.model.user_id == user_id))
             )
         if desc:
-            angents = angents.order_by(cls.model.getter_by(orderby).desc())
+            agents = agents.order_by(cls.model.getter_by(orderby).desc())
         else:
-            angents = angents.order_by(cls.model.getter_by(orderby).asc())
-        count = angents.count()
-        angents = angents.paginate(page_number, items_per_page)
-        return list(angents.dicts()), count
+            agents = agents.order_by(cls.model.getter_by(orderby).asc())
+        count = agents.count()
+        agents = agents.paginate(page_number, items_per_page)
+        return list(agents.dicts()), count
    
 
 def completion(tenant_id, agent_id, question, session_id=None, stream=True, **kwargs):
@@ -173,6 +173,19 @@ def completion(tenant_id, agent_id, question, session_id=None, stream=True, **kw
             conv.reference = []
         conv.reference.append({"chunks": [], "doc_aggs": []})
 
+        kwargs_changed = False
+        if kwargs:
+            query = canvas.get_preset_param()
+            if query:
+                for ele in query:
+                    if ele["key"] in kwargs:
+                        if ele["value"] != kwargs[ele["key"]]:
+                            ele["value"] = kwargs[ele["key"]]
+                            kwargs_changed = True
+        if kwargs_changed:
+            conv.dsl = json.loads(str(canvas))
+            API4ConversationService.update_by_id(session_id, {"dsl": conv.dsl})
+
     final_ans = {"reference": [], "content": ""}
     if stream:
         try:
@@ -281,8 +294,22 @@ def completionOpenAI(tenant_id, agent_id, question, session_id=None, stream=True
             "source": "agent",
             "dsl": cvs.dsl
         }
+        canvas.messages.append({"role": "user", "content": question, "id": message_id})
+        canvas.add_user_input(question)
+
         API4ConversationService.save(**conv)
         conv = API4Conversation(**conv)
+        if not conv.message:
+            conv.message = []
+        conv.message.append({
+            "role": "user",
+            "content": question,
+            "id": message_id
+        })
+        
+        if not conv.reference:
+            conv.reference = []
+        conv.reference.append({"chunks": [], "doc_aggs": []})
             
     # Handle existing session
     else:
@@ -318,7 +345,7 @@ def completionOpenAI(tenant_id, agent_id, question, session_id=None, stream=True
     if stream:
         try:
             completion_tokens = 0
-            for ans in canvas.run(stream=True):
+            for ans in canvas.run(stream=True, bypass_begin=True):
                 if ans.get("running_status"):
                     completion_tokens += len(tiktokenenc.encode(ans.get("content", "")))
                     yield "data: " + json.dumps(
@@ -381,7 +408,7 @@ def completionOpenAI(tenant_id, agent_id, question, session_id=None, stream=True
     else:  # Non-streaming mode
         try:
             all_answer_content = ""
-            for answer in canvas.run(stream=False):
+            for answer in canvas.run(stream=False, bypass_begin=True):
                 if answer.get("running_status"):
                     continue
                 

api/db/services/common_service.py

@@ -254,7 +254,7 @@ class CommonService:
         # Returns:
         #     Number of records deleted
         return cls.model.delete().where(cls.model.id == pid).execute()
-
+    
     @classmethod
     @DB.connection_context()
     def delete_by_ids(cls, pids):

api/db/services/conversation_service.py

@@ -90,17 +90,18 @@ def completion(tenant_id, chat_id, question, name="New session", session_id=None
             "user_id": kwargs.get("user_id", "")
         }
         ConversationService.save(**conv)
-        yield "data:" + json.dumps({"code": 0, "message": "",
-                                    "data": {
-                                        "answer": conv["message"][0]["content"],
-                                        "reference": {},
-                                        "audio_binary": None,
-                                        "id": None,
-                                        "session_id": session_id
-                                    }},
-                                   ensure_ascii=False) + "\n\n"
-        yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
-        return
+        if stream:
+            yield "data:" + json.dumps({"code": 0, "message": "",
+                                        "data": {
+                                            "answer": conv["message"][0]["content"],
+                                            "reference": {},
+                                            "audio_binary": None,
+                                            "id": None,
+                                            "session_id": session_id
+                                        }},
+                                    ensure_ascii=False) + "\n\n"
+            yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
+            return
 
     conv = ConversationService.query(id=session_id, dialog_id=chat_id)
     if not conv:
@@ -123,6 +124,8 @@ def completion(tenant_id, chat_id, question, name="New session", session_id=None
     message_id = msg[-1].get("id")
     e, dia = DialogService.get_by_id(conv.dialog_id)
 
+    kb_ids = kwargs.get("kb_ids",[])
+    dia.kb_ids = list(set(dia.kb_ids + kb_ids))
     if not conv.reference:
         conv.reference = []
     conv.message.append({"role": "assistant", "content": "", "id": message_id})

api/db/services/dialog_service.py

@@ -127,9 +127,71 @@ def chat_solo(dialog, messages, stream=True):
         yield {"answer": answer, "reference": {}, "audio_binary": tts(tts_mdl, answer), "prompt": "", "created_at": time.time()}
 
 
+def get_models(dialog):
+    embd_mdl, chat_mdl, rerank_mdl, tts_mdl = None, None, None, None
+    kbs = KnowledgebaseService.get_by_ids(dialog.kb_ids)
+    embedding_list = list(set([kb.embd_id for kb in kbs]))
+    if len(embedding_list) > 1:
+        raise Exception("**ERROR**: Knowledge bases use different embedding models.")
+
+    if embedding_list:
+        embd_mdl = LLMBundle(dialog.tenant_id, LLMType.EMBEDDING, embedding_list[0])
+        if not embd_mdl:
+            raise LookupError("Embedding model(%s) not found" % embedding_list[0])
+
+    if llm_id2llm_type(dialog.llm_id) == "image2text":
+        chat_mdl = LLMBundle(dialog.tenant_id, LLMType.IMAGE2TEXT, dialog.llm_id)
+    else:
+        chat_mdl = LLMBundle(dialog.tenant_id, LLMType.CHAT, dialog.llm_id)
+
+    if dialog.rerank_id:
+        rerank_mdl = LLMBundle(dialog.tenant_id, LLMType.RERANK, dialog.rerank_id)
+
+    if dialog.prompt_config.get("tts"):
+        tts_mdl = LLMBundle(dialog.tenant_id, LLMType.TTS)
+    return kbs, embd_mdl, rerank_mdl, chat_mdl, tts_mdl
+
+
+BAD_CITATION_PATTERNS = [
+    re.compile(r"\(\s*ID\s*[: ]*\s*(\d+)\s*\)"),  # (ID: 12)
+    re.compile(r"\[\s*ID\s*[: ]*\s*(\d+)\s*\]"),  # [ID: 12]
+    re.compile(r"【\s*ID\s*[: ]*\s*(\d+)\s*】"),  # 【ID: 12】
+    re.compile(r"ref\s*(\d+)", flags=re.IGNORECASE),  # ref12、REF 12
+]
+
+
+def repair_bad_citation_formats(answer: str, kbinfos: dict, idx: set):
+    max_index = len(kbinfos["chunks"])
+
+    def safe_add(i):
+        if 0 <= i < max_index:
+            idx.add(i)
+            return True
+        return False
+
+    def find_and_replace(pattern, group_index=1, repl=lambda i: f"ID:{i}", flags=0):
+        nonlocal answer
+
+        def replacement(match):
+            try:
+                i = int(match.group(group_index))
+                if safe_add(i):
+                    return f"[{repl(i)}]"
+            except Exception:
+                pass
+            return match.group(0)
+
+        answer = re.sub(pattern, replacement, answer, flags=flags)
+
+    for pattern in BAD_CITATION_PATTERNS:
+        find_and_replace(pattern)
+
+    return answer, idx
+
+
 def chat(dialog, messages, stream=True, **kwargs):
     assert messages[-1]["role"] == "user", "The last content of this conversation is not from user."
-    if not dialog.kb_ids:
+    if not dialog.kb_ids and not dialog.prompt_config.get("tavily_api_key"):
         for ans in chat_solo(dialog, messages, stream):
             yield ans
         return
@@ -154,45 +216,19 @@ def chat(dialog, messages, stream=True, **kwargs):
             langfuse.trace = langfuse_tracer.trace(name=f"{dialog.name}-{llm_model_config['llm_name']}")
 
     check_langfuse_tracer_ts = timer()
-
-    kbs = KnowledgebaseService.get_by_ids(dialog.kb_ids)
-    embedding_list = list(set([kb.embd_id for kb in kbs]))
-    if len(embedding_list) != 1:
-        yield {"answer": "**ERROR**: Knowledge bases use different embedding models.", "reference": []}
-        return {"answer": "**ERROR**: Knowledge bases use different embedding models.", "reference": []}
-
-    embedding_model_name = embedding_list[0]
+    kbs, embd_mdl, rerank_mdl, chat_mdl, tts_mdl = get_models(dialog)
+    toolcall_session, tools = kwargs.get("toolcall_session"), kwargs.get("tools")
+    if toolcall_session and tools:
+        chat_mdl.bind_tools(toolcall_session, tools)
+    bind_models_ts = timer()
 
     retriever = settings.retrievaler
-
     questions = [m["content"] for m in messages if m["role"] == "user"][-3:]
     attachments = kwargs["doc_ids"].split(",") if "doc_ids" in kwargs else None
     if "doc_ids" in messages[-1]:
         attachments = messages[-1]["doc_ids"]
-
-    create_retriever_ts = timer()
-
-    embd_mdl = LLMBundle(dialog.tenant_id, LLMType.EMBEDDING, embedding_model_name)
-    if not embd_mdl:
-        raise LookupError("Embedding model(%s) not found" % embedding_model_name)
-
-    bind_embedding_ts = timer()
-
-    if llm_id2llm_type(dialog.llm_id) == "image2text":
-        chat_mdl = LLMBundle(dialog.tenant_id, LLMType.IMAGE2TEXT, dialog.llm_id)
-    else:
-        chat_mdl = LLMBundle(dialog.tenant_id, LLMType.CHAT, dialog.llm_id)
-        toolcall_session, tools = kwargs.get("toolcall_session"), kwargs.get("tools")
-        if toolcall_session and tools:
-            chat_mdl.bind_tools(toolcall_session, tools)
-
-    bind_llm_ts = timer()
-
     prompt_config = dialog.prompt_config
     field_map = KnowledgebaseService.get_field_map(dialog.kb_ids)
-    tts_mdl = None
-    if prompt_config.get("tts"):
-        tts_mdl = LLMBundle(dialog.tenant_id, LLMType.TTS)
     # try to use sql if field mapping is good to go
     if field_map:
         logging.debug("Use SQL to retrieval:{}".format(questions[-1]))
@@ -217,26 +253,18 @@ def chat(dialog, messages, stream=True, **kwargs):
     if prompt_config.get("cross_languages"):
         questions = [cross_languages(dialog.tenant_id, dialog.llm_id, questions[0], prompt_config["cross_languages"])]
 
+    if prompt_config.get("keyword", False):
+        questions[-1] += keyword_extraction(chat_mdl, questions[-1])
+
     refine_question_ts = timer()
 
-    rerank_mdl = None
-    if dialog.rerank_id:
-        rerank_mdl = LLMBundle(dialog.tenant_id, LLMType.RERANK, dialog.rerank_id)
-
-    bind_reranker_ts = timer()
-    generate_keyword_ts = bind_reranker_ts
     thought = ""
     kbinfos = {"total": 0, "chunks": [], "doc_aggs": []}
 
     if "knowledge" not in [p["key"] for p in prompt_config["parameters"]]:
         knowledges = []
     else:
-        if prompt_config.get("keyword", False):
-            questions[-1] += keyword_extraction(chat_mdl, questions[-1])
-            generate_keyword_ts = timer()
-
         tenant_ids = list(set([kb.tenant_id for kb in kbs]))
-
         knowledges = []
         if prompt_config.get("reasoning", False):
             reasoner = DeepResearcher(
@@ -252,21 +280,22 @@ def chat(dialog, messages, stream=True, **kwargs):
                 elif stream:
                     yield think
         else:
-            kbinfos = retriever.retrieval(
-                " ".join(questions),
-                embd_mdl,
-                tenant_ids,
-                dialog.kb_ids,
-                1,
-                dialog.top_n,
-                dialog.similarity_threshold,
-                dialog.vector_similarity_weight,
-                doc_ids=attachments,
-                top=dialog.top_k,
-                aggs=False,
-                rerank_mdl=rerank_mdl,
-                rank_feature=label_question(" ".join(questions), kbs),
-            )
+            if embd_mdl:
+                kbinfos = retriever.retrieval(
+                    " ".join(questions),
+                    embd_mdl,
+                    tenant_ids,
+                    dialog.kb_ids,
+                    1,
+                    dialog.top_n,
+                    dialog.similarity_threshold,
+                    dialog.vector_similarity_weight,
+                    doc_ids=attachments,
+                    top=dialog.top_k,
+                    aggs=False,
+                    rerank_mdl=rerank_mdl,
+                    rank_feature=label_question(" ".join(questions), kbs),
+                )
             if prompt_config.get("tavily_api_key"):
                 tav = Tavily(prompt_config["tavily_api_key"])
                 tav_res = tav.retrieve_chunks(" ".join(questions))
@@ -302,41 +331,8 @@ def chat(dialog, messages, stream=True, **kwargs):
     if "max_tokens" in gen_conf:
         gen_conf["max_tokens"] = min(gen_conf["max_tokens"], max_tokens - used_token_count)
 
-    def repair_bad_citation_formats(answer: str, kbinfos: dict, idx: set):
-        max_index = len(kbinfos["chunks"])
-
-        def safe_add(i):
-            if 0 <= i < max_index:
-                idx.add(i)
-                return True
-            return False
-
-        def find_and_replace(pattern, group_index=1, repl=lambda i: f"##{i}$$", flags=0):
-            nonlocal answer
-            for match in re.finditer(pattern, answer, flags=flags):
-                try:
-                    i = int(match.group(group_index))
-                    if safe_add(i):
-                        answer = answer.replace(match.group(0), repl(i))
-                except Exception:
-                    continue
-
-        find_and_replace(r"\(\s*ID:\s*(\d+)\s*\)")  # (ID: 12)
-        find_and_replace(r"ID[: ]+(\d+)")  # ID: 12, ID 12
-        find_and_replace(r"\$\$(\d+)\$\$")  # $$12$$
-        find_and_replace(r"\$\[(\d+)\]\$")  # $[12]$
-        find_and_replace(r"\$\$(\d+)\${2,}")  # $$12$$$$
-        find_and_replace(r"\$(\d+)\$")  # $12$
-        find_and_replace(r"(#{2,})(\d+)(\${2,})", group_index=2)  # 2+ # and 2+ $
-        find_and_replace(r"(#{2,})(\d+)(#{1,})", group_index=2)  # 2+ # and 1+ #
-        find_and_replace(r"##(\d+)#{2,}")  # ##12###
-        find_and_replace(r"【(\d+)】")  # 【12】
-        find_and_replace(r"ref\s*(\d+)", flags=re.IGNORECASE)  # ref12, ref 12, REF 12
-
-        return answer, idx
-
     def decorate_answer(answer):
-        nonlocal prompt_config, knowledges, kwargs, kbinfos, prompt, retrieval_ts, questions, langfuse_tracer
+        nonlocal embd_mdl, prompt_config, knowledges, kwargs, kbinfos, prompt, retrieval_ts, questions, langfuse_tracer
 
         refs = []
         ans = answer.split("</think>")
@@ -346,9 +342,8 @@ def chat(dialog, messages, stream=True, **kwargs):
             answer = ans[1]
 
         if knowledges and (prompt_config.get("quote", True) and kwargs.get("quote", True)):
-            answer = re.sub(r"##[ij]\$\$", "", answer, flags=re.DOTALL)
             idx = set([])
-            if not re.search(r"##[0-9]+\$\$", answer):
+            if embd_mdl and not re.search(r"\[ID:([0-9]+)\]", answer):
                 answer, idx = retriever.insert_citations(
                     answer,
                     [ck["content_ltks"] for ck in kbinfos["chunks"]],
@@ -358,7 +353,7 @@ def chat(dialog, messages, stream=True, **kwargs):
                     vtweight=dialog.vector_similarity_weight,
                 )
             else:
-                for match in re.finditer(r"##([0-9]+)\$\$", answer):
+                for match in re.finditer(r"\[ID:([0-9]+)\]", answer):
                     i = int(match.group(1))
                     if i < len(kbinfos["chunks"]):
                         idx.add(i)
@@ -383,13 +378,9 @@ def chat(dialog, messages, stream=True, **kwargs):
         total_time_cost = (finish_chat_ts - chat_start_ts) * 1000
         check_llm_time_cost = (check_llm_ts - chat_start_ts) * 1000
         check_langfuse_tracer_cost = (check_langfuse_tracer_ts - check_llm_ts) * 1000
-        create_retriever_time_cost = (create_retriever_ts - check_langfuse_tracer_ts) * 1000
-        bind_embedding_time_cost = (bind_embedding_ts - create_retriever_ts) * 1000
-        bind_llm_time_cost = (bind_llm_ts - bind_embedding_ts) * 1000
-        refine_question_time_cost = (refine_question_ts - bind_llm_ts) * 1000
-        bind_reranker_time_cost = (bind_reranker_ts - refine_question_ts) * 1000
-        generate_keyword_time_cost = (generate_keyword_ts - bind_reranker_ts) * 1000
-        retrieval_time_cost = (retrieval_ts - generate_keyword_ts) * 1000
+        bind_embedding_time_cost = (bind_models_ts - check_langfuse_tracer_ts) * 1000
+        refine_question_time_cost = (refine_question_ts - bind_models_ts) * 1000
+        retrieval_time_cost = (retrieval_ts - refine_question_ts) * 1000
         generate_result_time_cost = (finish_chat_ts - retrieval_ts) * 1000
 
         tk_num = num_tokens_from_string(think + answer)
@@ -400,12 +391,8 @@ def chat(dialog, messages, stream=True, **kwargs):
             f"  - Total: {total_time_cost:.1f}ms\n"
             f"  - Check LLM: {check_llm_time_cost:.1f}ms\n"
             f"  - Check Langfuse tracer: {check_langfuse_tracer_cost:.1f}ms\n"
-            f"  - Create retriever: {create_retriever_time_cost:.1f}ms\n"
-            f"  - Bind embedding: {bind_embedding_time_cost:.1f}ms\n"
-            f"  - Bind LLM: {bind_llm_time_cost:.1f}ms\n"
-            f"  - Multi-turn optimization: {refine_question_time_cost:.1f}ms\n"
-            f"  - Bind reranker: {bind_reranker_time_cost:.1f}ms\n"
-            f"  - Generate keyword: {generate_keyword_time_cost:.1f}ms\n"
+            f"  - Bind models: {bind_embedding_time_cost:.1f}ms\n"
+            f"  - Query refinement(LLM): {refine_question_time_cost:.1f}ms\n"
             f"  - Retrieval: {retrieval_time_cost:.1f}ms\n"
             f"  - Generate answer: {generate_result_time_cost:.1f}ms\n\n"
             "## Token usage:\n"
@@ -569,7 +556,7 @@ def tts(tts_mdl, text):
     return binascii.hexlify(bin).decode("utf-8")
 
 
-def ask(question, kb_ids, tenant_id):
+def ask(question, kb_ids, tenant_id, chat_llm_name=None):
     kbs = KnowledgebaseService.get_by_ids(kb_ids)
     embedding_list = list(set([kb.embd_id for kb in kbs]))
 
@@ -577,7 +564,7 @@ def ask(question, kb_ids, tenant_id):
     retriever = settings.retrievaler if not is_knowledge_graph else settings.kg_retrievaler
 
     embd_mdl = LLMBundle(tenant_id, LLMType.EMBEDDING, embedding_list[0])
-    chat_mdl = LLMBundle(tenant_id, LLMType.CHAT)
+    chat_mdl = LLMBundle(tenant_id, LLMType.CHAT, chat_llm_name)
     max_tokens = chat_mdl.max_length
     tenant_ids = list(set([kb.tenant_id for kb in kbs]))
     kbinfos = retriever.retrieval(question, embd_mdl, tenant_ids, kb_ids, 1, 12, 0.1, 0.3, aggs=False, rank_feature=label_question(question, kbs))

api/db/services/document_service.py

@@ -27,6 +27,7 @@ import xxhash
 from peewee import fn
 
 from api import settings
+from api.constants import IMG_BASE64_PREFIX
 from api.db import FileType, LLMType, ParserType, StatusEnum, TaskStatus, UserTenantRole
 from api.db.db_models import DB, Document, Knowledgebase, Task, Tenant, UserTenant
 from api.db.db_utils import bulk_insert_into_db
@@ -34,7 +35,7 @@ from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.utils import current_timestamp, get_format_time, get_uuid
 from rag.nlp import rag_tokenizer, search
-from rag.settings import get_svr_queue_name
+from rag.settings import get_svr_queue_name, SVR_CONSUMER_GROUP_NAME
 from rag.utils.redis_conn import REDIS_CONN
 from rag.utils.storage_factory import STORAGE_IMPL
 from rag.utils.doc_store_conn import OrderByExpr
@@ -147,7 +148,26 @@ class DocumentService(CommonService):
     def remove_document(cls, doc, tenant_id):
         cls.clear_chunk_num(doc.id)
         try:
+            page = 0
+            page_size = 1000
+            all_chunk_ids = []
+            while True:
+                chunks = settings.docStoreConn.search(["img_id"], [], {"doc_id": doc.id}, [], OrderByExpr(),
+                                                      page * page_size, page_size, search.index_name(tenant_id),
+                                                      [doc.kb_id])
+                chunk_ids = settings.docStoreConn.getChunkIds(chunks)
+                if not chunk_ids:
+                    break
+                all_chunk_ids.extend(chunk_ids)
+                page += 1
+            for cid in all_chunk_ids:
+                if STORAGE_IMPL.obj_exist(doc.kb_id, cid):
+                    STORAGE_IMPL.rm(doc.kb_id, cid)
+            if doc.thumbnail and not doc.thumbnail.startswith(IMG_BASE64_PREFIX):
+                if STORAGE_IMPL.obj_exist(doc.kb_id, doc.thumbnail):
+                    STORAGE_IMPL.rm(doc.kb_id, doc.thumbnail)
             settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id)
+
             graph_source = settings.docStoreConn.getFields(
                 settings.docStoreConn.search(["source_id"], [], {"kb_id": doc.kb_id, "knowledge_graph_kwd": ["graph"]}, [], OrderByExpr(), 0, 1, search.index_name(tenant_id), [doc.kb_id]), ["source_id"]
             )
@@ -464,7 +484,8 @@ class DocumentService(CommonService):
                     if t.progress == -1:
                         bad += 1
                     prg += t.progress if t.progress >= 0 else 0
-                    msg.append(t.progress_msg)
+                    if t.progress_msg.strip():
+                        msg.append(t.progress_msg)
                     if t.task_type == "raptor":
                         has_raptor = True
                     elif t.task_type == "graphrag":
@@ -494,6 +515,8 @@ class DocumentService(CommonService):
                     info["progress"] = prg
                 if msg:
                     info["progress_msg"] = msg
+                else:
+                    info["progress_msg"] = "%d tasks are ahead in the queue..."%get_queue_length(priority)
                 cls.update_by_id(d["id"], info)
             except Exception as e:
                 if str(e).find("'0'") < 0:
@@ -542,6 +565,11 @@ def queue_raptor_o_graphrag_tasks(doc, ty, priority):
     assert REDIS_CONN.queue_product(get_svr_queue_name(priority), message=task), "Can't access Redis. Please check the Redis' status."
 
 
+def get_queue_length(priority):
+    group_info = REDIS_CONN.queue_info(get_svr_queue_name(priority), SVR_CONSUMER_GROUP_NAME)
+    return int(group_info.get("lag", 0))
+
+
 def doc_upload_and_parse(conversation_id, file_objs, user_id):
     from api.db.services.api_service import API4ConversationService
     from api.db.services.conversation_service import ConversationService

api/db/services/file_service.py

@@ -21,6 +21,7 @@ from concurrent.futures import ThreadPoolExecutor
 from flask_login import current_user
 from peewee import fn
 
+from api.constants import FILE_NAME_LEN_LIMIT
 from api.db import KNOWLEDGEBASE_FOLDER_NAME, FileSource, FileType, ParserType
 from api.db.db_models import DB, Document, File, File2Document, Knowledgebase
 from api.db.services import duplicate_name
@@ -412,8 +413,8 @@ class FileService(CommonService):
                 MAX_FILE_NUM_PER_USER = int(os.environ.get("MAX_FILE_NUM_PER_USER", 0))
                 if MAX_FILE_NUM_PER_USER > 0 and DocumentService.get_doc_count(kb.tenant_id) >= MAX_FILE_NUM_PER_USER:
                     raise RuntimeError("Exceed the maximum file number of a free user!")
-                if len(file.filename.encode("utf-8")) >= 128:
-                    raise RuntimeError("Exceed the maximum length of file name!")
+                if len(file.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+                    raise RuntimeError(f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.")
 
                 filename = duplicate_name(DocumentService.query, name=file.filename, kb_id=kb.id)
                 filetype = filename_type(filename)
@@ -492,4 +493,3 @@ class FileService(CommonService):
         if re.search(r"\.(eml)$", filename):
             return ParserType.EMAIL.value
         return default
-

api/db/services/llm_service.py

@@ -169,7 +169,7 @@ class TenantLLMService(CommonService):
             return 0
 
         llm_map = {
-            LLMType.EMBEDDING.value: tenant.embd_id,
+            LLMType.EMBEDDING.value: tenant.embd_id if not llm_name else llm_name,
             LLMType.SPEECH2TEXT.value: tenant.asr_id,
             LLMType.IMAGE2TEXT.value: tenant.img2txt_id,
             LLMType.CHAT.value: tenant.llm_id if not llm_name else llm_name,
@@ -235,7 +235,8 @@ class LLMBundle:
             generation = self.trace.generation(name="encode", model=self.llm_name, input={"texts": texts})
 
         embeddings, used_tokens = self.mdl.encode(texts)
-        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens):
+        llm_name = getattr(self, "llm_name", None)
+        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens, llm_name):
             logging.error("LLMBundle.encode can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
 
         if self.langfuse:
@@ -248,7 +249,8 @@ class LLMBundle:
             generation = self.trace.generation(name="encode_queries", model=self.llm_name, input={"query": query})
 
         emd, used_tokens = self.mdl.encode_queries(query)
-        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens):
+        llm_name = getattr(self, "llm_name", None)
+        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens, llm_name):
             logging.error("LLMBundle.encode_queries can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
 
         if self.langfuse:

api/db/services/search_service.py

@@ -0,0 +1,110 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from datetime import datetime
+
+from peewee import fn
+
+from api.db import StatusEnum
+from api.db.db_models import DB, Search, User
+from api.db.services.common_service import CommonService
+from api.utils import current_timestamp, datetime_format
+
+
+class SearchService(CommonService):
+    model = Search
+
+    @classmethod
+    def save(cls, **kwargs):
+        kwargs["create_time"] = current_timestamp()
+        kwargs["create_date"] = datetime_format(datetime.now())
+        kwargs["update_time"] = current_timestamp()
+        kwargs["update_date"] = datetime_format(datetime.now())
+        obj = cls.model.create(**kwargs)
+        return obj
+
+    @classmethod
+    @DB.connection_context()
+    def accessible4deletion(cls, search_id, user_id) -> bool:
+        search = (
+            cls.model.select(cls.model.id)
+            .where(
+                cls.model.id == search_id,
+                cls.model.created_by == user_id,
+                cls.model.status == StatusEnum.VALID.value,
+            )
+            .first()
+        )
+        return search is not None
+
+    @classmethod
+    @DB.connection_context()
+    def get_detail(cls, search_id):
+        fields = [
+            cls.model.id,
+            cls.model.avatar,
+            cls.model.tenant_id,
+            cls.model.name,
+            cls.model.description,
+            cls.model.created_by,
+            cls.model.search_config,
+            cls.model.update_time,
+            User.nickname,
+            User.avatar.alias("tenant_avatar"),
+        ]
+        search = (
+            cls.model.select(*fields)
+            .join(User, on=((User.id == cls.model.tenant_id) & (User.status == StatusEnum.VALID.value)))
+            .where((cls.model.id == search_id) & (cls.model.status == StatusEnum.VALID.value))
+            .first()
+            .to_dict()
+        )
+        return search
+
+    @classmethod
+    @DB.connection_context()
+    def get_by_tenant_ids(cls, joined_tenant_ids, user_id, page_number, items_per_page, orderby, desc, keywords):
+        fields = [
+            cls.model.id,
+            cls.model.avatar,
+            cls.model.tenant_id,
+            cls.model.name,
+            cls.model.description,
+            cls.model.created_by,
+            cls.model.status,
+            cls.model.update_time,
+            cls.model.create_time,
+            User.nickname,
+            User.avatar.alias("tenant_avatar"),
+        ]
+        query = (
+            cls.model.select(*fields)
+            .join(User, on=(cls.model.tenant_id == User.id))
+            .where(((cls.model.tenant_id.in_(joined_tenant_ids)) | (cls.model.tenant_id == user_id)) & (cls.model.status == StatusEnum.VALID.value))
+        )
+
+        if keywords:
+            query = query.where(fn.LOWER(cls.model.name).contains(keywords.lower()))
+        if desc:
+            query = query.order_by(cls.model.getter_by(orderby).desc())
+        else:
+            query = query.order_by(cls.model.getter_by(orderby).asc())
+
+        count = query.count()
+
+        if page_number and items_per_page:
+            query = query.paginate(page_number, items_per_page)
+
+        return list(query.dicts()), count

api/db/services/task_service.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import os
 import random
 import xxhash
@@ -256,36 +257,55 @@ class TaskService(CommonService):
     @DB.connection_context()
     def update_progress(cls, id, info):
         """Update the progress information for a task.
-    
+
         This method updates both the progress message and completion percentage of a task.
         It handles platform-specific behavior (macOS vs others) and uses database locking
         when necessary to ensure thread safety.
-    
+
+        Update Rules:
+            - progress_msg: Always appends the new message to the existing one, and trims the result to max 3000 lines.
+            - progress: Only updates if the current progress is not -1 AND
+                        (the new progress is -1 OR greater than the existing progress),
+                        to avoid overwriting valid progress with invalid or regressive values.
+
         Args:
             id (str): The unique identifier of the task to update.
             info (dict): Dictionary containing progress information with keys:
                         - progress_msg (str, optional): Progress message to append
                         - progress (float, optional): Progress percentage (0.0 to 1.0)
         """
+        task = cls.model.get_by_id(id)
+        if not task:
+            logging.warning("Update_progress error: task not found")
+            return
+
         if os.environ.get("MACOS"):
             if info["progress_msg"]:
-                task = cls.model.get_by_id(id)
                 progress_msg = trim_header_by_lines(task.progress_msg + "\n" + info["progress_msg"], 3000)
                 cls.model.update(progress_msg=progress_msg).where(cls.model.id == id).execute()
             if "progress" in info:
-                cls.model.update(progress=info["progress"]).where(
-                    cls.model.id == id
+                prog = info["progress"]
+                cls.model.update(progress=prog).where(
+                    (cls.model.id == id) &
+                    (
+                        (cls.model.progress != -1) &
+                        ((prog == -1) | (prog > cls.model.progress))
+                    )
                 ).execute()
             return
 
         with DB.lock("update_progress", -1):
             if info["progress_msg"]:
-                task = cls.model.get_by_id(id)
                 progress_msg = trim_header_by_lines(task.progress_msg + "\n" + info["progress_msg"], 3000)
                 cls.model.update(progress_msg=progress_msg).where(cls.model.id == id).execute()
             if "progress" in info:
-                cls.model.update(progress=info["progress"]).where(
-                    cls.model.id == id
+                prog = info["progress"]
+                cls.model.update(progress=prog).where(
+                    (cls.model.id == id) &
+                    (
+                        (cls.model.progress != -1) &
+                        ((prog == -1) | (prog > cls.model.progress))
+                    )
                 ).execute()
 
 

api/db/services/user_service.py

@@ -15,6 +15,7 @@
 #
 import hashlib
 from datetime import datetime
+import logging
 
 import peewee
 from werkzeug.security import generate_password_hash, check_password_hash
@@ -39,6 +40,30 @@ class UserService(CommonService):
     """
     model = User
 
+    @classmethod
+    @DB.connection_context()
+    def query(cls, cols=None, reverse=None, order_by=None, **kwargs):
+        if 'access_token' in kwargs:
+            access_token = kwargs['access_token']
+            
+            # Reject empty, None, or whitespace-only access tokens
+            if not access_token or not str(access_token).strip():
+                logging.warning("UserService.query: Rejecting empty access_token query")
+                return cls.model.select().where(cls.model.id == "INVALID_EMPTY_TOKEN")  # Returns empty result
+            
+            # Reject tokens that are too short (should be UUID, 32+ chars)
+            if len(str(access_token).strip()) < 32:
+                logging.warning(f"UserService.query: Rejecting short access_token query: {len(str(access_token))} chars")
+                return cls.model.select().where(cls.model.id == "INVALID_SHORT_TOKEN")  # Returns empty result
+            
+            # Reject tokens that start with "INVALID_" (from logout)
+            if str(access_token).startswith("INVALID_"):
+                logging.warning("UserService.query: Rejecting invalidated access_token")
+                return cls.model.select().where(cls.model.id == "INVALID_LOGOUT_TOKEN")  # Returns empty result
+        
+        # Call parent query method for valid requests
+        return super().query(cols=cols, reverse=reverse, order_by=order_by, **kwargs)
+
     @classmethod
     @DB.connection_context()
     def filter_by_id(cls, user_id):

api/ragflow_server.py

@@ -18,9 +18,9 @@
 # from beartype.claw import beartype_all  # <-- you didn't sign up for this
 # beartype_all(conf=BeartypeConf(violation_type=UserWarning))    # <-- emit warnings from all code
 
-from api.utils.log_utils import initRootLogger
+from api.utils.log_utils import init_root_logger
 from plugin import GlobalPluginManager
-initRootLogger("ragflow_server")
+init_root_logger("ragflow_server")
 
 import logging
 import os
@@ -28,7 +28,6 @@ import signal
 import sys
 import time
 import traceback
-from concurrent.futures import ThreadPoolExecutor
 import threading
 import uuid
 
@@ -125,8 +124,16 @@ if __name__ == '__main__':
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
 
-    thread = ThreadPoolExecutor(max_workers=1)
-    thread.submit(update_progress)
+    def delayed_start_update_progress():
+        logging.info("Starting update_progress thread (delayed)")
+        t = threading.Thread(target=update_progress, daemon=True)
+        t.start()
+
+    if RuntimeConfig.DEBUG:
+        if os.environ.get("WERKZEUG_RUN_MAIN") == "true":
+            threading.Timer(1.0, delayed_start_update_progress).start()
+    else:
+        threading.Timer(1.0, delayed_start_update_progress).start()
 
     # start http server
     try:

api/settings.py

@@ -15,6 +15,7 @@
 #
 import json
 import os
+import secrets
 from datetime import date
 from enum import Enum, IntEnum
 
@@ -73,6 +74,25 @@ SANDBOX_HOST = None
 
 BUILTIN_EMBEDDING_MODELS = ["BAAI/bge-large-zh-v1.5@BAAI", "maidalun1020/bce-embedding-base_v1@Youdao"]
 
+def get_or_create_secret_key():
+    secret_key = os.environ.get("RAGFLOW_SECRET_KEY")
+    if secret_key and len(secret_key) >= 32:
+        return secret_key
+    
+    # Check if there's a configured secret key
+    configured_key = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("secret_key")
+    if configured_key and configured_key != str(date.today()) and len(configured_key) >= 32:
+        return configured_key
+    
+    # Generate a new secure key and warn about it
+    import logging
+    new_key = secrets.token_hex(32)
+    logging.warning(
+        "SECURITY WARNING: Using auto-generated SECRET_KEY. "
+        f"Generated key: {new_key}"
+    )
+    return new_key
+
 
 def init_settings():
     global LLM, LLM_FACTORY, LLM_BASE_URL, LIGHTEN, DATABASE_TYPE, DATABASE, FACTORY_LLM_INFOS, REGISTER_ENABLED
@@ -121,7 +141,7 @@ def init_settings():
     HOST_IP = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("host", "127.0.0.1")
     HOST_PORT = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("http_port")
 
-    SECRET_KEY = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("secret_key", str(date.today()))
+    SECRET_KEY = get_or_create_secret_key()
 
     global AUTHENTICATION_CONF, CLIENT_AUTHENTICATION, HTTP_APP_KEY, GITHUB_OAUTH, FEISHU_OAUTH, OAUTH_CONFIG
     # authentication

api/utils/api_utils.py

@@ -428,11 +428,11 @@ def verify_embedding_availability(embd_id: str, tenant_id: str) -> tuple[bool, R
     """
     Verifies availability of an embedding model for a specific tenant.
 
-    Implements a four-stage validation process:
-    1. Model identifier parsing and validation
-    2. System support verification
-    3. Tenant authorization check
-    4. Database operation error handling
+    Performs comprehensive verification through:
+    1. Identifier Parsing: Decomposes embd_id into name and factory components
+    2. System Verification: Checks model registration in LLMService
+    3. Tenant Authorization: Validates tenant-specific model assignments
+    4. Built-in Model Check: Confirms inclusion in predefined system models
 
     Args:
         embd_id (str): Unique identifier for the embedding model in format "model_name@factory"
@@ -460,14 +460,15 @@ def verify_embedding_availability(embd_id: str, tenant_id: str) -> tuple[bool, R
     """
     try:
         llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(embd_id)
-        if not LLMService.query(llm_name=llm_name, fid=llm_factory, model_type="embedding"):
-            return False, get_error_argument_result(f"Unsupported model: <{embd_id}>")
+        in_llm_service = bool(LLMService.query(llm_name=llm_name, fid=llm_factory, model_type="embedding"))
 
-        # Tongyi-Qianwen is added to TenantLLM by default, but remains unusable with empty api_key
         tenant_llms = TenantLLMService.get_my_llms(tenant_id=tenant_id)
         is_tenant_model = any(llm["llm_name"] == llm_name and llm["llm_factory"] == llm_factory and llm["model_type"] == "embedding" for llm in tenant_llms)
 
         is_builtin_model = embd_id in settings.BUILTIN_EMBEDDING_MODELS
+        if not (is_builtin_model or is_tenant_model or in_llm_service):
+            return False, get_error_argument_result(f"Unsupported model: <{embd_id}>")
+
         if not (is_builtin_model or is_tenant_model):
             return False, get_error_argument_result(f"Unauthorized model: <{embd_id}>")
     except OperationalError as e:

api/utils/file_utils.py

@@ -158,7 +158,7 @@ def filename_type(filename):
     if re.match(r".*\.(eml|doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|html|sql)$", filename):
         return FileType.DOC.value
 
-    if re.match(r".*\.(wav|flac|ape|alac|wavpack|wv|mp3|aac|ogg|vorbis|opus|mp3)$", filename):
+    if re.match(r".*\.(wav|flac|ape|alac|wavpack|wv|mp3|aac|ogg|vorbis|opus)$", filename):
         return FileType.AURAL.value
 
     if re.match(r".*\.(jpg|jpeg|png|tif|gif|pcx|tga|exif|fpx|svg|psd|cdr|pcd|dxf|ufo|eps|ai|raw|WMF|webp|avif|apng|icon|ico|mpg|mpeg|avi|rm|rmvb|mov|wmv|asf|dat|asx|wvx|mpe|mpa|mp4)$", filename):

api/utils/log_utils.py

@@ -30,7 +30,7 @@ def get_project_base_directory():
     )
     return PROJECT_BASE
 
-def initRootLogger(logfile_basename: str, log_format: str = "%(asctime)-15s %(levelname)-8s %(process)d %(message)s"):
+def init_root_logger(logfile_basename: str, log_format: str = "%(asctime)-15s %(levelname)-8s %(process)d %(message)s"):
     global initialized_root_logger
     if initialized_root_logger:
         return
@@ -77,4 +77,11 @@ def initRootLogger(logfile_basename: str, log_format: str = "%(asctime)-15s %(le
         pkg_logger.setLevel(pkg_level)
 
     msg = f"{logfile_basename} log path: {log_path}, log levels: {pkg_levels}"
-    logger.info(msg)
+    logger.info(msg)
+
+
+def log_exception(e, *args):
+    logging.exception(e)
+    for a in args:
+        logging.error(str(a))
+    raise e

api/utils/t_crypt.py

@@ -35,6 +35,6 @@ def crypt(line):
 
 
 if __name__ == "__main__":
-    pswd = crypt(sys.argv[1])
-    print(pswd)
-    print(decrypt(pswd))
+    passwd = crypt(sys.argv[1])
+    print(passwd)
+    print(decrypt(passwd))

api/utils/validation_utils.py

@@ -312,7 +312,7 @@ class PermissionEnum(StrEnum):
     team = auto()
 
 
-class ChunkMethodnEnum(StrEnum):
+class ChunkMethodEnum(StrEnum):
     naive = auto()
     book = auto()
     email = auto()
@@ -382,8 +382,7 @@ class CreateDatasetReq(Base):
     description: str | None = Field(default=None, max_length=65535)
     embedding_model: Annotated[str, StringConstraints(strip_whitespace=True, max_length=255), Field(default="", serialization_alias="embd_id")]
     permission: PermissionEnum = Field(default=PermissionEnum.me, min_length=1, max_length=16)
-    chunk_method: ChunkMethodnEnum = Field(default=ChunkMethodnEnum.naive, min_length=1, max_length=32, serialization_alias="parser_id")
-    pagerank: int = Field(default=0, ge=0, le=100)
+    chunk_method: ChunkMethodEnum = Field(default=ChunkMethodEnum.naive, min_length=1, max_length=32, serialization_alias="parser_id")
     parser_config: ParserConfig | None = Field(default=None)
 
     @field_validator("avatar")
@@ -539,6 +538,7 @@ class CreateDatasetReq(Base):
 class UpdateDatasetReq(CreateDatasetReq):
     dataset_id: str = Field(...)
     name: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=DATASET_NAME_LIMIT), Field(default="")]
+    pagerank: int = Field(default=0, ge=0, le=100)
 
     @field_validator("dataset_id", mode="before")
     @classmethod

conf/llm_factories.json

@@ -360,6 +360,12 @@
                     "max_tokens": 8192,
                     "model_type": "embedding"
                 },
+                {
+                    "llm_name": "text-embedding-v4",
+                    "tags": "TEXT EMBEDDING,8K",
+                    "max_tokens": 8192,
+                    "model_type": "embedding"
+                },
                 {
                     "llm_name": "qwen-vl-max",
                     "tags": "LLM,CHAT,IMAGE2TEXT",
@@ -3195,6 +3201,12 @@
             "tags": "TEXT EMBEDDING, TEXT RE-RANK",
             "status": "1",
             "llm": [
+                {
+                    "llm_name": "voyage-multimodal-3",
+                    "tags": "TEXT EMBEDDING,Chat,IMAGE2TEXT,32000",
+                    "max_tokens": 32000,
+                    "model_type": "embedding"
+                },
                 {
                     "llm_name": "voyage-large-2-instruct",
                     "tags": "TEXT EMBEDDING,16000",

conf/service_conf.yaml

@@ -9,6 +9,7 @@ mysql:
   port: 5455
   max_connections: 900
   stale_timeout: 300
+  max_allowed_packet: 1073741824
 minio:
   user: 'rag_flow'
   password: 'infini_rag_flow'
@@ -28,7 +29,6 @@ redis:
   db: 1
   password: 'infini_rag_flow'
   host: 'localhost:6379'
-
 # postgres:
 #   name: 'rag_flow'
 #   user: 'rag_flow'
@@ -58,6 +58,11 @@ redis:
 #   secret: 'secret'
 #   tenant_id: 'tenant_id'
 #   container_name: 'container_name'
+# The OSS object storage uses the MySQL configuration above by default. If you need to switch to another object storage service, please uncomment and configure the following parameters.
+# opendal:
+#   scheme: 'mysql'  # Storage type, such as s3, oss, azure, etc.
+#   config:
+#     oss_table: 'your_table_name'
 # user_default_llm:
 #   factory: 'Tongyi-Qianwen'
 #   api_key: 'sk-xxxxxxxxxxxxx'

deepdoc/parser/docx_parser.py

@@ -69,7 +69,7 @@ class RAGFlowDocxParser:
         max_type = max(max_type.items(), key=lambda x: x[1])[0]
 
         colnm = len(df.iloc[0, :])
-        hdrows = [0]  # header is not nessesarily appear in the first line
+        hdrows = [0]  # header is not necessarily appear in the first line
         if max_type == "Nu":
             for r in range(1, len(df)):
                 tys = Counter([blockType(str(df.iloc[r, j]))

deepdoc/parser/figure_parser.py

@@ -21,7 +21,7 @@ from rag.app.picture import vision_llm_chunk as picture_vision_llm_chunk
 from rag.prompts import vision_llm_figure_describe_prompt
 
 
-def vision_figure_parser_figure_data_wraper(figures_data_without_positions):
+def vision_figure_parser_figure_data_wrapper(figures_data_without_positions):
     return [
         (
             (figure_data[1], [figure_data[0]]),

deepdoc/parser/pdf_parser.py

@@ -61,7 +61,7 @@ class RAGFlowPdfParser:
 
         self.ocr = OCR()
         self.parallel_limiter = None
-        if PARALLEL_DEVICES is not None and PARALLEL_DEVICES > 1:
+        if PARALLEL_DEVICES > 1:
             self.parallel_limiter = [trio.CapacityLimiter(1) for _ in range(PARALLEL_DEVICES)]
 
         if hasattr(self, "model_speciess"):
@@ -180,13 +180,13 @@ class RAGFlowPdfParser:
         return fea
 
     @staticmethod
-    def sort_X_by_page(arr, threashold):
+    def sort_X_by_page(arr, threshold):
         # sort using y1 first and then x1
         arr = sorted(arr, key=lambda r: (r["page_number"], r["x0"], r["top"]))
         for i in range(len(arr) - 1):
             for j in range(i, -1, -1):
                 # restore the order using th
-                if abs(arr[j + 1]["x0"] - arr[j]["x0"]) < threashold \
+                if abs(arr[j + 1]["x0"] - arr[j]["x0"]) < threshold \
                         and arr[j + 1]["top"] < arr[j]["top"] \
                         and arr[j + 1]["page_number"] == arr[j]["page_number"]:
                     tmp = arr[j]
@@ -264,13 +264,13 @@ class RAGFlowPdfParser:
         for b in self.boxes:
             if b.get("layout_type", "") != "table":
                 continue
-            ii = Recognizer.find_overlapped_with_threashold(b, rows, thr=0.3)
+            ii = Recognizer.find_overlapped_with_threshold(b, rows, thr=0.3)
             if ii is not None:
                 b["R"] = ii
                 b["R_top"] = rows[ii]["top"]
                 b["R_bott"] = rows[ii]["bottom"]
 
-            ii = Recognizer.find_overlapped_with_threashold(
+            ii = Recognizer.find_overlapped_with_threshold(
                 b, headers, thr=0.3)
             if ii is not None:
                 b["H_top"] = headers[ii]["top"]
@@ -285,7 +285,7 @@ class RAGFlowPdfParser:
                 b["C_left"] = clmns[ii]["x0"]
                 b["C_right"] = clmns[ii]["x1"]
 
-            ii = Recognizer.find_overlapped_with_threashold(b, spans, thr=0.3)
+            ii = Recognizer.find_overlapped_with_threshold(b, spans, thr=0.3)
             if ii is not None:
                 b["H_top"] = spans[ii]["top"]
                 b["H_bott"] = spans[ii]["bottom"]

deepdoc/parser/ppt_parser.py

@@ -63,7 +63,7 @@ class RAGFlowPptParser:
             if shape_type == 6:
                 texts = []
                 for p in sorted(shape.shapes, key=lambda x: (x.top // 10, x.left)):
-                    t = self.__extract_texts(p)
+                    t = self.__extract(p)
                     if t:
                         texts.append(t)
                 return "\n".join(texts)

deepdoc/vision/layout_recognizer.py

@@ -106,7 +106,7 @@ class LayoutRecognizer(Recognizer):
                         bxs.pop(i)
                         continue
 
-                    ii = self.find_overlapped_with_threashold(bxs[i], lts_,
+                    ii = self.find_overlapped_with_threshold(bxs[i], lts_,
                                                               thr=0.4)
                     if ii is None:  # belong to nothing
                         bxs[i]["layout_type"] = ""

deepdoc/vision/ocr.py

@@ -529,31 +529,30 @@ class OCR:
                         "rag/res/deepdoc")
                 
                 # Append muti-gpus task to the list
-                if PARALLEL_DEVICES is not None and PARALLEL_DEVICES > 0:
+                if PARALLEL_DEVICES > 0:
                     self.text_detector = []
                     self.text_recognizer = []
                     for device_id in range(PARALLEL_DEVICES):
                         self.text_detector.append(TextDetector(model_dir, device_id))
                         self.text_recognizer.append(TextRecognizer(model_dir, device_id))
                 else:
-                    self.text_detector = [TextDetector(model_dir, 0)]
-                    self.text_recognizer = [TextRecognizer(model_dir, 0)]
+                    self.text_detector = [TextDetector(model_dir)]
+                    self.text_recognizer = [TextRecognizer(model_dir)]
 
             except Exception:
                 model_dir = snapshot_download(repo_id="InfiniFlow/deepdoc",
                                               local_dir=os.path.join(get_project_base_directory(), "rag/res/deepdoc"),
                                               local_dir_use_symlinks=False)
                 
-                if PARALLEL_DEVICES is not None:
-                    assert PARALLEL_DEVICES > 0, "Number of devices must be >= 1"
+                if PARALLEL_DEVICES > 0:
                     self.text_detector = []
                     self.text_recognizer = []
                     for device_id in range(PARALLEL_DEVICES):
                         self.text_detector.append(TextDetector(model_dir, device_id))
                         self.text_recognizer.append(TextRecognizer(model_dir, device_id))
                 else:
-                    self.text_detector = [TextDetector(model_dir, 0)]
-                    self.text_recognizer = [TextRecognizer(model_dir, 0)]
+                    self.text_detector = [TextDetector(model_dir)]
+                    self.text_recognizer = [TextRecognizer(model_dir)]
 
         self.drop_score = 0.5
         self.crop_image_res_index = 0
@@ -589,7 +588,29 @@ class OCR:
             flags=cv2.INTER_CUBIC)
         dst_img_height, dst_img_width = dst_img.shape[0:2]
         if dst_img_height * 1.0 / dst_img_width >= 1.5:
-            dst_img = np.rot90(dst_img)
+            # Try original orientation
+            rec_result = self.text_recognizer[0]([dst_img])
+            text, score = rec_result[0][0]
+            best_score = score
+            best_img = dst_img
+
+            # Try clockwise 90° rotation
+            rotated_cw = np.rot90(dst_img, k=3)
+            rec_result = self.text_recognizer[0]([rotated_cw])
+            rotated_cw_text, rotated_cw_score = rec_result[0][0]
+            if rotated_cw_score > best_score:
+                best_score = rotated_cw_score
+                best_img = rotated_cw
+
+            # Try counter-clockwise 90° rotation
+            rotated_ccw = np.rot90(dst_img, k=1)
+            rec_result = self.text_recognizer[0]([rotated_ccw])
+            rotated_ccw_text, rotated_ccw_score = rec_result[0][0]
+            if rotated_ccw_score > best_score:
+                best_img = rotated_ccw
+
+            # Use the best image
+            dst_img = best_img
         return dst_img
 
     def sorted_boxes(self, dt_boxes):

deepdoc/vision/recognizer.py

@@ -52,20 +52,20 @@ class Recognizer:
         self.label_list = label_list
 
     @staticmethod
-    def sort_Y_firstly(arr, threashold):
+    def sort_Y_firstly(arr, threshold):
         def cmp(c1, c2):
             diff = c1["top"] - c2["top"]
-            if abs(diff) < threashold:
+            if abs(diff) < threshold:
                 diff = c1["x0"] - c2["x0"]
             return diff
         arr = sorted(arr, key=cmp_to_key(cmp))
         return arr
 
     @staticmethod
-    def sort_X_firstly(arr, threashold):
+    def sort_X_firstly(arr, threshold):
         def cmp(c1, c2):
             diff = c1["x0"] - c2["x0"]
-            if abs(diff) < threashold:
+            if abs(diff) < threshold:
                 diff = c1["top"] - c2["top"]
             return diff
         arr = sorted(arr, key=cmp_to_key(cmp))
@@ -133,7 +133,7 @@ class Recognizer:
 
     @staticmethod
     def layouts_cleanup(boxes, layouts, far=2, thr=0.7):
-        def notOverlapped(a, b):
+        def not_overlapped(a, b):
             return any([a["x1"] < b["x0"],
                         a["x0"] > b["x1"],
                         a["bottom"] < b["top"],
@@ -144,7 +144,7 @@ class Recognizer:
             j = i + 1
             while j < min(i + far, len(layouts)) \
                     and (layouts[i].get("type", "") != layouts[j].get("type", "")
-                         or notOverlapped(layouts[i], layouts[j])):
+                         or not_overlapped(layouts[i], layouts[j])):
                 j += 1
             if j >= min(i + far, len(layouts)):
                 i += 1
@@ -163,9 +163,9 @@ class Recognizer:
 
             area_i, area_i_1 = 0, 0
             for b in boxes:
-                if not notOverlapped(b, layouts[i]):
+                if not not_overlapped(b, layouts[i]):
                     area_i += Recognizer.overlapped_area(b, layouts[i], False)
-                if not notOverlapped(b, layouts[j]):
+                if not not_overlapped(b, layouts[j]):
                     area_i_1 += Recognizer.overlapped_area(b, layouts[j], False)
 
             if area_i > area_i_1:
@@ -239,15 +239,15 @@ class Recognizer:
                 e -= 1
             break
 
-        max_overlaped_i, max_overlaped = None, 0
+        max_overlapped_i, max_overlapped = None, 0
         for i in range(s, e):
             ov = Recognizer.overlapped_area(bxs[i], box)
-            if ov <= max_overlaped:
+            if ov <= max_overlapped:
                 continue
-            max_overlaped_i = i
-            max_overlaped = ov
+            max_overlapped_i = i
+            max_overlapped = ov
 
-        return max_overlaped_i
+        return max_overlapped_i
 
     @staticmethod
     def find_horizontally_tightest_fit(box, boxes):
@@ -264,7 +264,7 @@ class Recognizer:
         return min_i
 
     @staticmethod
-    def find_overlapped_with_threashold(box, boxes, thr=0.3):
+    def find_overlapped_with_threshold(box, boxes, thr=0.3):
         if not boxes:
             return
         max_overlapped_i, max_overlapped, _max_overlapped = None, thr, 0
@@ -408,18 +408,18 @@ class Recognizer:
 
     def __call__(self, image_list, thr=0.7, batch_size=16):
         res = []
-        imgs = []
+        images = []
         for i in range(len(image_list)):
             if not isinstance(image_list[i], np.ndarray):
-                imgs.append(np.array(image_list[i]))
+                images.append(np.array(image_list[i]))
             else:
-                imgs.append(image_list[i])
+                images.append(image_list[i])
 
-        batch_loop_cnt = math.ceil(float(len(imgs)) / batch_size)
+        batch_loop_cnt = math.ceil(float(len(images)) / batch_size)
         for i in range(batch_loop_cnt):
             start_index = i * batch_size
-            end_index = min((i + 1) * batch_size, len(imgs))
-            batch_image_list = imgs[start_index:end_index]
+            end_index = min((i + 1) * batch_size, len(images))
+            batch_image_list = images[start_index:end_index]
             inputs = self.preprocess(batch_image_list)
             logging.debug("preprocess")
             for ins in inputs:

deepdoc/vision/t_recognizer.py

@@ -84,13 +84,13 @@ def get_table_html(img, tb_cpns, ocr):
     clmns = LayoutRecognizer.layouts_cleanup(boxes, clmns, 5, 0.5)
 
     for b in boxes:
-        ii = LayoutRecognizer.find_overlapped_with_threashold(b, rows, thr=0.3)
+        ii = LayoutRecognizer.find_overlapped_with_threshold(b, rows, thr=0.3)
         if ii is not None:
             b["R"] = ii
             b["R_top"] = rows[ii]["top"]
             b["R_bott"] = rows[ii]["bottom"]
 
-        ii = LayoutRecognizer.find_overlapped_with_threashold(b, headers, thr=0.3)
+        ii = LayoutRecognizer.find_overlapped_with_threshold(b, headers, thr=0.3)
         if ii is not None:
             b["H_top"] = headers[ii]["top"]
             b["H_bott"] = headers[ii]["bottom"]
@@ -104,7 +104,7 @@ def get_table_html(img, tb_cpns, ocr):
             b["C_left"] = clmns[ii]["x0"]
             b["C_right"] = clmns[ii]["x1"]
 
-        ii = LayoutRecognizer.find_overlapped_with_threashold(b, spans, thr=0.3)
+        ii = LayoutRecognizer.find_overlapped_with_threshold(b, spans, thr=0.3)
         if ii is not None:
             b["H_top"] = spans[ii]["top"]
             b["H_bott"] = spans[ii]["bottom"]

docker/.env

@@ -129,6 +129,14 @@ TIMEZONE='Asia/Shanghai'
 # Note that neither `MAX_CONTENT_LENGTH` nor `client_max_body_size` sets the maximum size for files uploaded to an agent.
 # See https://ragflow.io/docs/dev/begin_component for details.
 
+# Controls how many documents are processed in a single batch.
+# Defaults to 4 if DOC_BULK_SIZE is not explicitly set.
+DOC_BULK_SIZE=${DOC_BULK_SIZE:-4}
+
+# Defines the number of items to process per batch when generating embeddings.
+# Defaults to 16 if EMBEDDING_BATCH_SIZE is not set in the environment.
+EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-16}
+
 # Log level for the RAGFlow's own and imported packages.
 # Available levels:
 # - `DEBUG`

docker/README.md

@@ -78,8 +78,8 @@ The [.env](./.env) file contains important environment variables for Docker.
 - `RAGFLOW-IMAGE`  
   The Docker image edition. Available editions:  
   
-  - `infiniflow/ragflow:v0.19.0-slim` (default): The RAGFlow Docker image without embedding models.  
-  - `infiniflow/ragflow:v0.19.0`: The RAGFlow Docker image with embedding models including:
+  - `infiniflow/ragflow:v0.19.1-slim` (default): The RAGFlow Docker image without embedding models.  
+  - `infiniflow/ragflow:v0.19.1`: The RAGFlow Docker image with embedding models including:
     - Built-in embedding models:
       - `BAAI/bge-large-zh-v1.5` 
       - `maidalun1020/bce-embedding-base_v1`
@@ -115,6 +115,16 @@ The [.env](./.env) file contains important environment variables for Docker.
 - `MAX_CONTENT_LENGTH`  
   The maximum file size for each uploaded file, in bytes. You can uncomment this line if you wish to change the 128M file size limit. After making the change, ensure you update `client_max_body_size` in nginx/nginx.conf correspondingly.
 
+### Doc bulk size
+
+- `DOC_BULK_SIZE`  
+  The number of document chunks processed in a single batch during document parsing. Defaults to `4`.
+
+### Embedding batch size
+
+- `EMBEDDING_BATCH_SIZE`  
+  The number of text chunks processed in a single batch during embedding vectorization. Defaults to `16`.
+
 ## 🐋 Service configuration
 
 [service_conf.yaml](./service_conf.yaml) specifies the system-level configuration for RAGFlow and is used by its API server and task executor. In a dockerized setup, this file is automatically created based on the [service_conf.yaml.template](./service_conf.yaml.template) file (replacing all environment variables by their values).

docker/docker-compose.yml

@@ -1,3 +1,4 @@
+
 include:
   - ./docker-compose-base.yml
 

docker/service_conf.yaml.template

@@ -20,7 +20,7 @@ es:
 os:
   hosts: 'http://${OS_HOST:-opensearch01}:9201'
   username: '${OS_USER:-admin}'
-  password: '${OPENSEARCHH_PASSWORD:-infini_rag_flow_OS_01}'
+  password: '${OPENSEARCH_PASSWORD:-infini_rag_flow_OS_01}'
 infinity:
   uri: '${INFINITY_HOST:-infinity}:23817'
   db_name: 'default_db'

docs/configurations.md

@@ -99,8 +99,8 @@ RAGFlow utilizes MinIO as its object storage solution, leveraging its scalabilit
 - `RAGFLOW-IMAGE`  
   The Docker image edition. Available editions:  
   
-  - `infiniflow/ragflow:v0.19.0-slim` (default): The RAGFlow Docker image without embedding models.  
-  - `infiniflow/ragflow:v0.19.0`: The RAGFlow Docker image with embedding models including:
+  - `infiniflow/ragflow:v0.19.1-slim` (default): The RAGFlow Docker image without embedding models.  
+  - `infiniflow/ragflow:v0.19.1`: The RAGFlow Docker image with embedding models including:
     - Built-in embedding models:
       - `BAAI/bge-large-zh-v1.5` 
       - `maidalun1020/bce-embedding-base_v1`

docs/develop/build_docker_image.mdx

@@ -77,7 +77,7 @@ After building the infiniflow/ragflow:nightly-slim image, you are ready to launc
 
 1. Edit Docker Compose Configuration
 
-Open the `docker/.env` file. Find the `RAGFLOW_IMAGE` setting and change the image reference from `infiniflow/ragflow:v0.19.0-slim` to `infiniflow/ragflow:nightly-slim` to use the pre-built image.
+Open the `docker/.env` file. Find the `RAGFLOW_IMAGE` setting and change the image reference from `infiniflow/ragflow:v0.19.1-slim` to `infiniflow/ragflow:nightly-slim` to use the pre-built image.
 
 
 2. Launch the Service

docs/develop/mcp/launch_mcp_server.md

@@ -42,10 +42,10 @@ You can start an MCP server either from source code or via Docker.
 ```bash
 # Launch the MCP server to work in self-host mode, run either of the following
 uv run mcp/server/server.py --host=127.0.0.1 --port=9382 --base_url=http://127.0.0.1:9380 --api_key=ragflow-xxxxx
-# uv run mcp/server/server.py --host=127.0.0.1 --port=9382 --base_url=http://127.0.0.1:9380 mode=self-host --api_key=ragflow-xxxxx
+# uv run mcp/server/server.py --host=127.0.0.1 --port=9382 --base_url=http://127.0.0.1:9380 --mode=self-host --api_key=ragflow-xxxxx
 
 # To launch the MCP server to work in host mode, run the following instead:
-# uv run mcp/server/server.py --host=127.0.0.1 --port=9382 --base_url=http://127.0.0.1:9380 mode=host
+# uv run mcp/server/server.py --host=127.0.0.1 --port=9382 --base_url=http://127.0.0.1:9380 --mode=host
 ```
 
 Where: 

docs/develop/mcp/mcp_client_example.md

@@ -1,16 +1,240 @@
 ---
 sidebar_position: 3
 slug: /mcp_client
+
 ---
 
-# RAGFlow MCP client example
+# RAGFlow MCP client examples
+
+Python and curl MCP client examples.
+
+------
+
+## Example MCP Python client
 
 We provide a *prototype* MCP client example for testing [here](https://github.com/infiniflow/ragflow/blob/main/mcp/client/client.py).
 
 :::danger IMPORTANT
-If your MCP server is running in host mode, include your acquired API key in your client's `headers` as shown below:
+If your MCP server is running in host mode, include your acquired API key in your client's `headers` when connecting asynchronously to it:
+
 ```python
 async with sse_client("http://localhost:9382/sse", headers={"api_key": "YOUR_KEY_HERE"}) as streams:
     # Rest of your code...
 ```
-:::
+
+Alternatively, to comply with [OAuth 2.1 Section 5](https://datatracker.ietf.org/doc/html/draft-ietf-oauth-v2-1-12#section-5), you can run the following code *instead* to connect to your MCP server:
+
+```python
+async with sse_client("http://localhost:9382/sse", headers={"Authorization": "YOUR_KEY_HERE"}) as streams:
+    # Rest of your code...
+```
+:::
+
+## Use curl to interact with the RAGFlow MCP server
+
+When interacting with the MCP server via HTTP requests, follow this initialization sequence:
+
+1. **The client sends an `initialize` request** with protocol version and capabilities.
+2. **The server replies with an `initialize` response**, including the supported protocol and capabilities.
+3. **The client confirms readiness with an `initialized` notification**.
+   _The connection is established between the client and the server, and further operations (such as tool listing) may proceed._
+
+:::tip NOTE
+For more information about this initialization process, see [here](https://modelcontextprotocol.io/docs/concepts/architecture#1-initialization). 
+:::
+
+In the following sections, we will walk you through a complete tool calling process.
+
+### 1. Obtain a session ID
+
+Each curl request with the MCP server must include a session ID:
+
+```bash
+$ curl -N -H "api_key: YOUR_API_KEY" http://127.0.0.1:9382/sse
+```
+
+:::tip NOTE
+See [here](../acquire_ragflow_api_key.md) for information about acquiring an API key.
+:::
+
+#### Transport
+
+The transport will stream messages such as tool results, server responses, and keep-alive pings.
+
+_The server returns the session ID:_
+
+```bash
+event: endpoint
+data: /messages/?session_id=5c6600ef61b845a788ddf30dceb25c54
+```
+
+### 2. Send an `Initialize` request
+
+The client sends an `initialize` request with protocol version and capabilities:
+
+```bash
+session_id="5c6600ef61b845a788ddf30dceb25c54" && \
+
+curl -X POST "http://127.0.0.1:9382/messages/?session_id=$session_id" \
+  -H "api_key: YOUR_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0",
+    "id": 1,
+    "method": "initialize",
+    "params": {
+      "protocolVersion": "1.0",
+      "capabilities": {},
+      "clientInfo": {
+        "name": "ragflow-mcp-client",
+        "version": "0.1"
+      }
+    }
+  }' && \
+```
+
+#### Transport
+
+_The server replies with an `initialize` response, including the supported protocol and capabilities:_
+
+```bash
+event: message
+data: {"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-03-26","capabilities":{"experimental":{"headers":{"host":"127.0.0.1:9382","user-agent":"curl/8.7.1","accept":"*/*","api_key":"ragflow-xxxxxxxxxxxx","accept-encoding":"gzip"}},"tools":{"listChanged":false}},"serverInfo":{"name":"ragflow-server","version":"1.9.4"}}}
+```
+
+### 3. Acknowledge readiness
+
+The client confirms readiness with an `initialized` notification:
+
+```bash
+curl -X POST "http://127.0.0.1:9382/messages/?session_id=$session_id" \
+  -H "api_key: YOUR_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0",
+    "method": "notifications/initialized",
+    "params": {}
+  }' && \
+```
+
+ _The connection is established between the client and the server, and further operations (such as tool listing) may proceed._
+
+### 4. Tool listing
+
+```bash
+curl -X POST "http://127.0.0.1:9382/messages/?session_id=$session_id" \
+  -H "api_key: YOUR_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0",
+    "id": 3,
+    "method": "tools/list",
+    "params": {}
+  }' && \
+```
+
+#### Transport
+
+```bash
+event: message
+data: {"jsonrpc":"2.0","id":3,"result":{"tools":[{"name":"ragflow_retrieval","description":"Retrieve relevant chunks from the RAGFlow retrieve interface based on the question, using the specified dataset_ids and optionally document_ids. Below is the list of all available datasets, including their descriptions and IDs. If you're unsure which datasets are relevant to the question, simply pass all dataset IDs to the function.","inputSchema":{"type":"object","properties":{"dataset_ids":{"type":"array","items":{"type":"string"}},"document_ids":{"type":"array","items":{"type":"string"}},"question":{"type":"string"}},"required":["dataset_ids","question"]}}]}}
+
+```
+
+### 5. Tool calling
+
+```bash
+curl -X POST "http://127.0.0.1:9382/messages/?session_id=$session_id" \
+  -H "api_key: YOUR_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0",
+    "id": 4,
+    "method": "tools/call",
+    "params": {
+      "name": "ragflow_retrieval",
+      "arguments": {
+        "question": "How to install neovim?",
+        "dataset_ids": ["DATASET_ID_HERE"],
+        "document_ids": []
+      }
+
+```
+
+#### Transport
+
+```bash
+event: message
+data: {"jsonrpc":"2.0","id":4,"result":{...}}
+
+```
+
+### A complete curl example
+
+```bash
+session_id="YOUR_SESSION_ID" && \
+
+# Step 1: Initialize request
+curl -X POST "http://127.0.0.1:9382/messages/?session_id=$session_id" \
+  -H "api_key: YOUR_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0",
+    "id": 1,
+    "method": "initialize",
+    "params": {
+      "protocolVersion": "1.0",
+      "capabilities": {},
+      "clientInfo": {
+        "name": "ragflow-mcp-client",
+        "version": "0.1"
+      }
+    }
+  }' && \
+
+sleep 2 && \
+
+# Step 2: Initialized notification
+curl -X POST "http://127.0.0.1:9382/messages/?session_id=$session_id" \
+  -H "api_key: YOUR_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0",
+    "method": "notifications/initialized",
+    "params": {}
+  }' && \
+
+sleep 2 && \
+
+# Step 3: Tool listing
+curl -X POST "http://127.0.0.1:9382/messages/?session_id=$session_id" \
+  -H "api_key: YOUR_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0",
+    "id": 3,
+    "method": "tools/list",
+    "params": {}
+  }' && \
+
+sleep 2 && \
+
+# Step 4: Tool call
+curl -X POST "http://127.0.0.1:9382/messages/?session_id=$session_id" \
+  -H "api_key: YOUR_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0",
+    "id": 4,
+    "method": "tools/call",
+    "params": {
+      "name": "ragflow_retrieval",
+      "arguments": {
+        "question": "How to install neovim?",
+        "dataset_ids": ["DATASET_ID_HERE"],
+        "document_ids": []
+      }
+    }
+  }'
+
+```

docs/faq.mdx

@@ -19,7 +19,7 @@ import TOCInline from '@theme/TOCInline';
 
 ### What sets RAGFlow apart from other RAG products?
 
-The "garbage in garbage out" status quo remains unchanged despite the fact that LLMs have advanced Natural Language Processing (NLP) significantly. In response, RAGFlow introduces two unique features compared to other Retrieval-Augmented Generation (RAG) products.
+The "garbage in garbage out" status quo remains unchanged despite the fact that LLMs have advanced Natural Language Processing (NLP) significantly. In its response, RAGFlow introduces two unique features compared to other Retrieval-Augmented Generation (RAG) products.
 
 - Fine-grained document parsing: Document parsing involves images and tables, with the flexibility for you to intervene as needed.
 - Traceable answers with reduced hallucinations: You can trust RAGFlow's responses as you can view the citations and references supporting them.
@@ -30,17 +30,17 @@ The "garbage in garbage out" status quo remains unchanged despite the fact that
 
 Each RAGFlow release is available in two editions:
 
-- **Slim edition**: excludes built-in embedding models and is identified by a **-slim** suffix added to the version name. Example: `infiniflow/ragflow:v0.19.0-slim`
-- **Full edition**: includes built-in embedding models and has no suffix added to the version name. Example: `infiniflow/ragflow:v0.19.0`
+- **Slim edition**: excludes built-in embedding models and is identified by a **-slim** suffix added to the version name. Example: `infiniflow/ragflow:v0.19.1-slim`
+- **Full edition**: includes built-in embedding models and has no suffix added to the version name. Example: `infiniflow/ragflow:v0.19.1`
 
 ---
 
 ### Which embedding models can be deployed locally?
 
-RAGFlow offers two Docker image editions, `v0.19.0-slim` and `v0.19.0`:  
+RAGFlow offers two Docker image editions, `v0.19.1-slim` and `v0.19.1`:  
   
-- `infiniflow/ragflow:v0.19.0-slim` (default): The RAGFlow Docker image without embedding models.  
-- `infiniflow/ragflow:v0.19.0`: The RAGFlow Docker image with embedding models including:
+- `infiniflow/ragflow:v0.19.1-slim` (default): The RAGFlow Docker image without embedding models.  
+- `infiniflow/ragflow:v0.19.1`: The RAGFlow Docker image with embedding models including:
   - Built-in embedding models:
     - `BAAI/bge-large-zh-v1.5`
     - `maidalun1020/bce-embedding-base_v1`
@@ -127,7 +127,19 @@ The corresponding APIs are now available. See the [RAGFlow HTTP API Reference](.
 
 ### Do you support stream output?
 
-Yes, we do.
+Yes, we do. Stream output is enabled by default in the chat assistant and agent. Note that you cannot disable stream output via RAGFlow's UI. To disable stream output in responses, use RAGFlow's Python or RESTful APIs:
+
+Python:
+
+- [Create chat completion](./references/python_api_reference.md#create-chat-completion)
+- [Converse with chat assistant](./references/python_api_reference.md#converse-with-chat-assistant)
+- [Converse with agent](./references/python_api_reference.md#converse-with-agent)
+
+RESTful:
+
+- [Create chat completion](./references/http_api_reference.md#create-chat-completion)
+- [Converse with chat assistant](./references/http_api_reference.md#converse-with-chat-assistant)
+- [Converse with agent](./references/http_api_reference.md#converse-with-agent)
 
 ---
 
@@ -488,4 +500,10 @@ To switch your document engine from Elasticsearch to [Infinity](https://github.c
 
 All uploaded files are stored in Minio, RAGFlow's object storage solution. For instance, if you upload your file directly to a knowledge base, it is located at `<knowledgebase_id>/filename`.
 
----
+---
+
+### How to tune batch size for document parsing and embedding?
+
+You can control the batch size for document parsing and embedding by setting the environment variables `DOC_BULK_SIZE` and `EMBEDDING_BATCH_SIZE`. Increasing these values may improve throughput for large-scale data processing, but will also increase memory usage. Adjust them according to your hardware resources.
+
+---

docs/guides/agent/agent_component_reference/begin.mdx

@@ -21,7 +21,7 @@ Click the component to display its **Configuration** window. Here, you can set a
 
 ### ID
 
-The ID is the unique identifier for the component within the workflow. Unlike the IDs of other components, the ID of the **Begin** component *cannot* be changed.
+The ID is the unique identifier for the component within the workflow. Unlike the IDs of other components, the ID of the **Begin** component _cannot_ be changed.
 
 ### Opening greeting
 
@@ -31,30 +31,36 @@ An opening greeting is the agent's first message to the user. It can be a welcom
 
 You can set global variables within the **Begin** component, which can be either required or optional. Once established, users will need to provide values for these variables when interacting or chatting with the agent. Click **+ Add variable** to add a global variable, each with the following attributes:
 
-- **Key**: *Required*  
+- **Key**: _Required_  
   The unique variable name.
-- **Name**: *Required*  
+- **Name**: _Required_  
   A descriptive name providing additional details about the variable.  
   For example, if **Key** is set to `lang`, you can set its **Name** to `Target language`.
-- **Type**: *Required*  
-  The type of the variable:  
+- **Type**: _Required_  
+  The type of the variable:
   - **line**: Accepts a single line of text without line breaks.
   - **paragraph**: Accepts multiple lines of text, including line breaks.
-  - **options**: Requires the user to select a value for this variable from a dropdown menu. And you are required to set *at least* one option for the dropdown menu.
+  - **options**: Requires the user to select a value for this variable from a dropdown menu. And you are required to set _at least_ one option for the dropdown menu.
   - **file**: Requires the user to upload one or multiple files.
   - **integer**: Accepts an integer as input.
   - **boolean**: Requires the user to toggle between on and off.
-- **Optional**: A toggle indicating whether the variable is optional. 
+- **Optional**: A toggle indicating whether the variable is optional.
 
 :::tip NOTE
 To pass in parameters from a client, call:
+
 - HTTP method [Converse with agent](../../../references/http_api_reference.md#converse-with-agent), or
 - Python method [Converse with agent](../../../references/python_api_reference.md#converse-with-agent).
-:::
+  :::
 
 :::danger IMPORTANT
+
 - If you set the key type as **file**, ensure the token count of the uploaded file does not exceed your model provider's maximum token limit; otherwise, the plain text in your file will be truncated and incomplete.
-- If your agent's **Begin** component takes a variable, you *cannot* embed it into a webpage.
+- If your agent's **Begin** component takes a variable, you _cannot_ embed it into a webpage.
+  :::
+
+:::note
+You can tune document parsing and embedding efficiency by setting the environment variables `DOC_BULK_SIZE` and `EMBEDDING_BATCH_SIZE`.
 :::
 
 ## Examples
@@ -71,7 +77,7 @@ As mentioned earlier, the **Begin** component is indispensable for an agent. Sti
 
 ### Is the uploaded file in a knowledge base?
 
-No. Files uploaded to an agent as input are not stored in a knowledge base and hence will not be processed using RAGFlow's built-in OCR, DLR or TSR models, or chunked using RAGFlow's built-in chunking methods. 
+No. Files uploaded to an agent as input are not stored in a knowledge base and hence will not be processed using RAGFlow's built-in OCR, DLR or TSR models, or chunked using RAGFlow's built-in chunking methods.
 
 ### How to upload a webpage or file from a URL?
 
@@ -81,8 +87,8 @@ If you set the type of a variable as **file**, your users will be able to upload
 
 ### File size limit for an uploaded file
 
-There is no *specific* file size limit for a file uploaded to an agent. However, note that model providers typically have a default or explicit maximum token setting, which can range from 8196 to 128k: The plain text part of the uploaded file will be passed in as the key value, but if the file's token count exceeds this limit, the string will be truncated and incomplete.
+There is no _specific_ file size limit for a file uploaded to an agent. However, note that model providers typically have a default or explicit maximum token setting, which can range from 8196 to 128k: The plain text part of the uploaded file will be passed in as the key value, but if the file's token count exceeds this limit, the string will be truncated and incomplete.
 
 :::tip NOTE
 The variables `MAX_CONTENT_LENGTH` in `/docker/.env` and `client_max_body_size` in `/docker/nginx/nginx.conf` set the file size limit for each upload to a knowledge base or **File Management**. These settings DO NOT apply in this scenario.
-:::
+:::

docs/guides/agent/agent_component_reference/code.mdx

@@ -0,0 +1,52 @@
+---
+sidebar_position: 13
+slug: /code_component
+---
+
+# Code component
+
+A component that enables users to integrate Python or JavaScript codes into their Agent for dynamic data processing.
+
+---
+
+## Scenarios
+
+A **Code** component is essential when you need to integrate complex code logic (Python or JavaScript) into your Agent for dynamic data processing.
+
+## Input variables
+
+You can specify multiple input sources for the **Code** component. Click **+ Add variable** in the **Input variables** section to include the desired input variables. 
+
+After defining an input variable, you are required to select from the dropdown menu:
+  - A component ID under **Component Output**, or 
+  - A global variable under **Begin input**, which is defined in the **Begin** component.
+
+## Coding field
+
+This field allows you to enter and edit your source code.
+
+### A Python code example
+
+```Python 
+    def main(arg1: str, arg2: str) -> dict:
+        return {
+            "result": arg1 + arg2,
+        }
+```
+
+### A JavaScript code example
+
+```JavaScript
+
+    const axios = require('axios');
+    async function main(args) {
+      try {
+        const response = await axios.get('https://github.com/infiniflow/ragflow');
+        console.log('Body:', response.data);
+      } catch (error) {
+        console.error('Error:', error.message);
+      }
+    }
+```
+
+

docs/guides/agent/agent_component_reference/note.mdx

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 13
+sidebar_position: 18
 slug: /note_component
 ---
 

docs/guides/agent/sandbox_quickstart.md

@@ -0,0 +1,116 @@
+---
+sidebar_position: 20
+slug: /sandbox_quickstart
+---
+
+# Sandbox quickstart
+
+A secure, pluggable code execution backend designed for RAGFlow and other applications requiring isolated code execution environments.
+
+## Features: 
+
+- Seamless RAGFlow Integration — Works out-of-the-box with the code component of RAGFlow.
+- High Security — Uses gVisor for syscall-level sandboxing to isolate execution.
+- Customisable Sandboxing — Modify seccomp profiles easily to tailor syscall restrictions.
+- Pluggable Runtime Support — Extendable to support any programming language runtime.
+- Developer Friendly — Quick setup with a convenient Makefile.
+
+## Architecture
+
+The architecture consists of isolated Docker base images for each supported language runtime, managed by the executor manager service. The executor manager orchestrates sandboxed code execution using gVisor for syscall interception and optional seccomp profiles for enhanced syscall filtering.
+
+## Prerequisites
+
+- Linux distribution compatible with gVisor.
+- gVisor installed and configured.
+- Docker version 24.0.0 or higher.
+- Docker Compose version 2.26.1 or higher (similar to RAGFlow requirements).
+- uv package and project manager installed.
+- (Optional) GNU Make for simplified command-line management.
+
+## Build Docker base images
+
+The sandbox uses isolated base images for secure containerised execution environments.
+
+Build the base images manually:
+
+```bash
+docker build -t sandbox-base-python:latest ./sandbox_base_image/python
+docker build -t sandbox-base-nodejs:latest ./sandbox_base_image/nodejs
+```
+
+Alternatively, build all base images at once using the Makefile:
+
+```bash
+make build
+```
+
+Next, build the executor manager image:
+
+```bash
+docker build -t sandbox-executor-manager:latest ./executor_manager
+```
+
+## Running with RAGFlow 
+
+1. Verify that gVisor is properly installed and operational.
+
+2. Configure the .env file located at docker/.env:
+
+- Uncomment sandbox-related environment variables.
+- Enable the sandbox profile at the bottom of the file.
+
+3. Add the following entry to your /etc/hosts file to resolve the executor manager service:
+
+```bash
+127.0.0.1 sandbox-executor-manager
+```
+
+4. Start the RAGFlow service as usual.
+
+## Running standalone
+
+### Manual setup
+
+1. Initialize the environment variables:
+
+```bash
+cp .env.example .env
+```
+
+2. Launch the sandbox services with Docker Compose:
+
+```bash
+docker compose -f docker-compose.yml up
+```
+
+3. Test the sandbox setup:
+
+```bash
+source .venv/bin/activate
+export PYTHONPATH=$(pwd)
+uv pip install -r executor_manager/requirements.txt
+uv run tests/sandbox_security_tests_full.py
+```
+
+### Using Makefile
+
+Run all setup, build, launch, and tests with a single command:
+
+```bash
+make
+```
+
+### Monitoring
+
+To follow logs of the executor manager container:
+
+```bash
+docker logs -f sandbox-executor-manager
+```
+
+Or use the Makefile shortcut:
+
+```bash
+make logs
+```

docs/guides/ai_search.md

@@ -9,7 +9,7 @@ Conduct an AI search.
 
 ---
 
-An AI search is a single-turn AI conversation using a predefined retrieval strategy (a hybrid search of weighted keyword similarity and weighted vector similarity) and the system's default chat model. It does not involve advanced RAG strategies like knowledge graph, auto-keyword, or auto-question. Retrieved chunks will be listed below the chat model's response.
+An AI search is a single-turn AI conversation using a predefined retrieval strategy (a hybrid search of weighted keyword similarity and weighted vector similarity) and the system's default chat model. It does not involve advanced RAG strategies like knowledge graph, auto-keyword, or auto-question. The related chunks are listed below the chat model's response in descending order based on their similarity scores. 
 
 ![](https://raw.githubusercontent.com/infiniflow/ragflow-docs/main/images/ai_search.jpg)
 

docs/guides/chat/start_chat.md

@@ -42,13 +42,13 @@ You start an AI conversation by creating an assistant.
    - **Rerank model** sets the reranker model to use. It is left empty by default.
      - If **Rerank model** is left empty, the hybrid score system uses keyword similarity and vector similarity, and the default weight assigned to the vector similarity component is 1-0.7=0.3.
      - If **Rerank model** is selected, the hybrid score system uses keyword similarity and reranker score, and the default weight assigned to the reranker score is 1-0.7=0.3.
-   - **Cross-language search**: Optional  
+   - [Cross-language search](../../references/glossary.mdx#cross-language-search): Optional  
      Select one or more target languages from the dropdown menu. The system’s default chat model will then translate your query into the selected target language(s). This translation ensures accurate semantic matching across languages, allowing you to retrieve relevant results regardless of language differences.  
      - When selecting target languages, please ensure that these languages are present in the knowledge base to guarantee an effective search.
      - If no target language is selected, the system will search only in the language of your query, which may cause relevant information in other languages to be missed.
    - **Variable** refers to the variables (keys) to be used in the system prompt. `{knowledge}` is a reserved variable. Click **Add** to add more variables for the system prompt.
       - If you are uncertain about the logic behind **Variable**, leave it *as-is*.
-      - As of v0.19.0, if you add custom variables here, the only way you can pass in their values is to call:
+      - As of v0.19.1, if you add custom variables here, the only way you can pass in their values is to call:
          - HTTP method [Converse with chat assistant](../../references/http_api_reference.md#converse-with-chat-assistant), or
          - Python method [Converse with chat assistant](../../references/python_api_reference.md#converse-with-chat-assistant).
 

docs/guides/dataset/autokeyword_autoquestion.mdx

@@ -0,0 +1,72 @@
+---
+sidebar_position: 3
+slug: /autokeyword_autoquestion
+---
+
+# Auto-keyword Auto-question
+import APITable from '@site/src/components/APITable';
+
+Use a chat model to generate keywords or questions from each chunk in the knowledge base.
+
+---
+
+When selecting a chunking method, you can also enable auto-keyword or auto-question generation to increase retrieval rates. This feature uses a chat model to produce a specified number of keywords and questions from each created chunk, generating an "additional layer of information" from the original content.
+
+:::caution WARNING
+Enabling this feature increases document indexing time and uses extra tokens, as all created chunks will be sent to the chat model for keyword or question generation.
+:::
+
+## What is Auto-keyword?
+
+Auto-keyword refers to the auto-keyword generation feature of RAGFlow. It uses a chat model to generate a set of keywords or synonyms from each chunk to correct errors and enhance retrieval accuracy. This feature is implemented as a slider under **Page rank** on the **Configuration** page of your knowledge base.
+
+**Values**:  
+
+- 0: (Default) Disabled.  
+- Between 3 and 5 (inclusive): Recommended if you have chunks of approximately 1,000 characters. 
+- 30 (maximum) 
+
+:::tip NOTE
+- If your chunk size increases, you can increase the value accordingly. Please note, as the value increases, the marginal benefit decreases. 
+- An Auto-keyword value must be an integer. If you set it to a non-integer, say 1.7, it will be rounded down to the nearest integer, which in this case is 1.
+:::
+
+## What is Auto-question?
+
+Auto-question is a feature of RAGFlow that automatically generates questions from chunks of data using a chat model. These questions (e.g. who, what, and why) also help correct errors and improve the matching of user queries. The feature usually works with FAQ retrieval scenarios involving product manuals or policy documents. And you can find this feature as a slider under **Page rank** on the **Configuration** page of your knowledge base.
+
+**Values**:
+
+- 0: (Default) Disabled.  
+- 1 or 2: Recommended if you have chunks of approximately 1,000 characters.  
+- 10 (maximum)
+
+:::tip NOTE
+- If your chunk size increases, you can increase the value accordingly. Please note, as the value increases, the marginal benefit decreases. 
+- An Auto-question value must be an integer. If you set it to a non-integer, say 1.7, it will be rounded down to the nearest integer, which in this case is 1.
+:::
+
+## Tips from the community
+
+The Auto-keyword or Auto-question values relate closely to the chunking size in your knowledge base. However, if you are new to this feature and unsure which value(s) to start with, the following are some value settings we gathered from our community. While they may not be accurate, they provide a starting point at the very least.
+
+```mdx-code-block
+<APITable>
+```
+
+| Use cases or typical scenarios                                      | Document volume/length          | Auto_keyword (0–30)        | Auto_question (0–10)       |
+|---------------------------------------------------------------------|---------------------------------|----------------------------|----------------------------|
+| Internal process guidance for employee handbook                     | Small, under 10 pages           | 0                          | 0                          |
+| Customer service FAQs                                               | Medium, 10–100 pages            | 3–7                        | 1–3                        |
+| Technical whitepapers: Development standards, protocol details      | Large, over 100 pages           | 2–4                        | 1–2                        |
+| Contracts / Regulations / Legal clause retrieval                    | Large, over 50 pages            | 2–5                        | 0–1                        |
+| Multi-repository layered new documents + old archive                | Many                            | Adjust as appropriate      |Adjust as appropriate       |
+| Social media comment pool: multilingual & mixed spelling            | Very large volume of short text | 8–12                       | 0                          |
+| Operational logs for troubleshooting                                | Very large volume of short text | 3–6                        | 0                          |
+| Marketing asset library: multilingual product descriptions          | Medium                          | 6–10                       | 1–2                        |
+| Training courses / eBooks                                           | Large                           | 2–5                        | 1–2                        |
+| Maintenance manual: equipment diagrams + steps                      | Medium                          | 3–7                        | 1–2                        |
+
+```mdx-code-block
+</APITable>
+```

docs/guides/dataset/configure_knowledge_base.md

@@ -41,7 +41,7 @@ RAGFlow offers multiple chunking template to facilitate chunking files of differ
 
 | **Template** | Description                                                           | File format                                                                                   |
 |--------------|-----------------------------------------------------------------------|-----------------------------------------------------------------------------------------------|
-| General      | Files are consecutively chunked based on a preset chunk token number. | DOCX, XLSX, XLS (Excel 97-2003), PPT, PDF, TXT, JPEG, JPG, PNG, TIF, GIF, CSV, JSON, EML, HTML |
+| General      | Files are consecutively chunked based on a preset chunk token number. | MD, MDX, DOCX, XLSX, XLS (Excel 97-2003), PPT, PDF, TXT, JPEG, JPG, PNG, TIF, GIF, CSV, JSON, EML, HTML |
 | Q&A          |                                                                       | XLSX, XLS (Excel 97-2003), CSV/TXT                                                             |
 | Resume       | Enterprise edition only. You can also try it out on demo.ragflow.io.  | DOCX, PDF, TXT                                                                                |
 | Manual       |                                                                       | PDF                                                                                           |
@@ -68,7 +68,7 @@ The following embedding models can be deployed locally:
 - maidalun1020/bce-embedding-base_v1
 
 :::danger IMPORTANT
-Please note these two embedding models support both English and Chinese. If your knowledge base contains other languages, the performance may be COMPROMISED.
+These two embedding models are optimized specifically for English and Chinese, so performance may be compromised if you use them to embed documents in other languages.
 :::
 
 ### Upload file
@@ -128,7 +128,7 @@ See [Run retrieval test](./run_retrieval_test.md) for details.
 
 ## Search for knowledge base
 
-As of RAGFlow v0.19.0, the search feature is still in a rudimentary form, supporting only knowledge base search by name.
+As of RAGFlow v0.19.1, the search feature is still in a rudimentary form, supporting only knowledge base search by name.
 
 ![search knowledge base](https://github.com/infiniflow/ragflow/assets/93570324/836ae94c-2438-42be-879e-c7ad2a59693e)
 

docs/guides/dataset/enable_excel2html.md

@@ -9,7 +9,7 @@ Convert complex Excel spreadsheets into HTML tables.
 
 ---
 
-When using the General chunking method, you can enable the **Excel to HTML** toggle to convert spreadsheet files into HTML tables. If it is disabled, spreadsheet tables will be represented as key-value pairs. For complex tables that cannot be simply represented this way, you must enable this feature.
+When using the **General** chunking method, you can enable the **Excel to HTML** toggle to convert spreadsheet files into HTML tables. If it is disabled, spreadsheet tables will be represented as key-value pairs. For complex tables that cannot be simply represented this way, you must enable this feature.
 
 :::caution WARNING
 The feature is disabled by default. If your knowledge base contains spreadsheets with complex tables and you do not enable this feature, RAGFlow will not throw an error but your tables are likely to be garbled.
@@ -22,7 +22,7 @@ Works with complex tables that cannot be represented as key-value pairs. Example
 ## Considerations
 
 - The Excel2HTML feature applies only to spreadsheet files (XLSX or XLS (Excel 97-2003)).
-- This feature is associated with the General chunking method. In other words, it is available *only when* you select the General chunking method.
+- This feature is associated with the **General** chunking method. In other words, it is available *only when* you select the **General** chunking method.
 - When this feature is enabled, spreadsheet tables with more than 12 rows will be split into chunks of 12 rows each.
 
 ## Procedure

docs/guides/dataset/enable_raptor.md

@@ -47,7 +47,7 @@ The RAPTOR feature is disabled by default. To enable it, manually switch on the
 
 ### Prompt
 
-The following prompt will be applied recursively for cluster summarization, with `{cluster_content}` serving as an internal parameter. We recommend that you keep it as-is for now. The design will be updated in due course.
+The following prompt will be applied *recursively* for cluster summarization, with `{cluster_content}` serving as an internal parameter. We recommend that you keep it as-is for now. The design will be updated in due course.
 
 ```
 Please summarize the following paragraphs... Paragraphs as following:

docs/guides/dataset/run_retrieval_test.md

@@ -62,7 +62,7 @@ Using a knowledge graph in a retrieval test will significantly increase the time
 
 ### Cross-language search
 
-To perform a cross-language search, select one or more target languages from the dropdown menu. The system’s default chat model will then translate your query entered in the Test text field into the selected target language(s). This translation ensures accurate semantic matching across languages, allowing you to retrieve relevant results regardless of language differences.
+To perform a [cross-language search](../../references/glossary.mdx#cross-language-search), select one or more target languages from the dropdown menu. The system’s default chat model will then translate your query entered in the Test text field into the selected target language(s). This translation ensures accurate semantic matching across languages, allowing you to retrieve relevant results regardless of language differences.
 
 :::tip NOTE
 - When selecting target languages, please ensure that these languages are present in the knowledge base to guarantee an effective search.

docs/guides/dataset/select_pdf_parser.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 2
+sidebar_position: 1
 slug: /select_pdf_parser
 ---
 
@@ -33,7 +33,7 @@ RAGFlow isn't one-size-fits-all. It is built for flexibility and supports deeper
 
 2. Select the option that works best with your scenario:
 
-  - DeepDoc: (Default) The default visual model for OCR, TSR, and DLR tasks, which is time-consuming.
+  - DeepDoc: (Default) The default visual model performing OCR, TSR, and DLR tasks on PDFs, which can be time-consuming.
   - Naive: Skip OCR, TSR, and DLR tasks if *all* your PDFs are plain text.
   - A third-party visual model provided by a specific model provider.
 

docs/guides/dataset/set_metadata.md

@@ -25,4 +25,4 @@ Ensure that your metadata is in JSON format; otherwise, your updates will not be
 
 ### Can I set metadata for multiple documents at once?
 
-No, RAGFlow does not support batch metadata setting. If you still consider this feature essential, please [raise an issue](https://github.com/infiniflow/ragflow/issues) explaining your use case and its importance.
+No, you must set metadata *individually* for each document, as RAGFlow does not support batch setting of metadata. If you still consider this feature essential, please [raise an issue](https://github.com/infiniflow/ragflow/issues) explaining your use case and its importance.

docs/guides/dataset/set_page_rank.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 3
+sidebar_position: 2
 slug: /set_page_rank
 ---
 

docs/guides/dataset/use_tag_sets.md

@@ -5,7 +5,7 @@ slug: /use_tag_sets
 
 # Use tag set
 
-Use a tag set to tag chunks in your datasets.
+Use a tag set to auto-tag chunks in your datasets.
 
 ---
 
@@ -21,7 +21,7 @@ The auto-tagging feature is *unavailable* on the [Infinity](https://github.com/i
 
 Auto-tagging applies in situations where chunks are so similar to each other that the intended chunks cannot be distinguished from the rest. For example, when you have a few chunks about iPhone and a majority about iPhone case or iPhone accessaries, it becomes difficult to retrieve those chunks about iPhone without additional information.
 
-## Create tag set
+## 1. Create tag set
 
 You can consider a tag set as a closed set, and the tags to attach to the chunks in your dataset (knowledge base) are *exclusively* from the specified tag set. You use a tag set to "inform" RAGFlow which chunks to tag and which tags to apply.
 
@@ -41,6 +41,10 @@ As a rule of thumb, consider including the following entries in your tag table:
 
 ### Create a tag set
 
+:::danger IMPORTANT
+A tag set is *not* involved in document indexing or retrieval. Do not specify a tag set when configuring your chat assistant or agent.
+:::
+
 1. Click **+ Create knowledge base** to create a knowledge base.
 2. Navigate to the **Configuration** page of the created knowledge base and choose **Tag** as the default chunking method.
 3. Navigate to the **Dataset** page and upload and parse your table file in XLSX, CSV, or TXT formats.  
@@ -49,11 +53,7 @@ As a rule of thumb, consider including the following entries in your tag table:
 4. Click the **Table** tab to view the tag frequency table:  
    ![Image](https://github.com/user-attachments/assets/af91d10c-5ea5-491f-ab21-3803d5ebf59f)
 
-:::danger IMPORTANT
-A tag set is *not* involved in document indexing or retrieval. Do not specify a tag set when configuring your chat assistant or agent.
-:::
-
-## Tag chunks
+## 2. Tag chunks
 
 Once a tag set is created, you can apply it to your dataset:
 
@@ -67,7 +67,7 @@ If the tag set is missing from the dropdown, check that it has been created or c
 3. Re-parse your documents to start the auto-tagging process.  
    _In an AI chat scenario using auto-tagged datasets, each query will be tagged using the corresponding tag set(s) and chunks with these tags will have a higher chance to be retrieved._
 
-## Update tag set
+## 3. Update tag set
 
 Creating a tag set is *not* for once and for all. Oftentimes, you may find it necessary to update or delete existing tags or add new entries. 
 

docs/guides/manage_files.md

@@ -87,4 +87,4 @@ RAGFlow's file management allows you to download an uploaded file:
 
 ![download_file](https://github.com/infiniflow/ragflow/assets/93570324/cf3b297f-7d9b-4522-bf5f-4f45743e4ed5)
 
-> As of RAGFlow v0.19.0, bulk download is not supported, nor can you download an entire folder. 
+> As of RAGFlow v0.19.1, bulk download is not supported, nor can you download an entire folder. 

docs/guides/tracing.mdx

@@ -18,7 +18,7 @@ RAGFlow ships with a built-in [Langfuse](https://langfuse.com) integration so th
 Langfuse stores traces, spans and prompt payloads in a purpose-built observability backend and offers filtering and visualisations on top.  
 
 :::info NOTE
-• RAGFlow **≥ 0.19.0** (contains the Langfuse connector)  
+• RAGFlow **≥ 0.19.1** (contains the Langfuse connector)  
 • A Langfuse workspace (cloud or self-hosted) with a _Project Public Key_ and _Secret Key_
 :::
 

docs/guides/upgrade_ragflow.mdx

@@ -66,16 +66,16 @@ To upgrade RAGFlow, you must upgrade **both** your code **and** your Docker imag
    git clone https://github.com/infiniflow/ragflow.git
    ```
 
-2. Switch to the latest, officially published release, e.g., `v0.19.0`:
+2. Switch to the latest, officially published release, e.g., `v0.19.1`:
 
    ```bash
-   git checkout -f v0.19.0
+   git checkout -f v0.19.1
    ```
 
 3. Update **ragflow/docker/.env** as follows:
 
    ```bash
-   RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0
+   RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1
    ```
 
 4. Update the RAGFlow image and restart RAGFlow:
@@ -92,10 +92,10 @@ To upgrade RAGFlow, you must upgrade **both** your code **and** your Docker imag
 1. From an environment with Internet access, pull the required Docker image.
 2. Save the Docker image to a **.tar** file.
    ```bash
-   docker save -o ragflow.v0.19.0.tar infiniflow/ragflow:v0.19.0
+   docker save -o ragflow.v0.19.1.tar infiniflow/ragflow:v0.19.1
    ```
 3. Copy the **.tar** file to the target server.
 4. Load the **.tar** file into Docker:
    ```bash
-   docker load -i ragflow.v0.19.0.tar
+   docker load -i ragflow.v0.19.1.tar
    ```

docs/quickstart.mdx

@@ -29,7 +29,7 @@ If you are on an ARM platform, follow [this guide](./develop/build_docker_image.
 - RAM ≥ 16 GB;
 - Disk ≥ 50 GB;
 - Docker ≥ 24.0.0 & Docker Compose ≥ v2.26.1.
-- [gVisor](https://gvisor.dev/docs/user_guide/install/): Required only if you intend to use the code executor (sandbox) feature of RAGFlow.
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): Required only if you intend to use the code executor ([sandbox](https://github.com/infiniflow/ragflow/tree/main/sandbox)) feature of RAGFlow.
 
 :::tip NOTE
 If you have not installed Docker on your local machine (Windows, Mac, or Linux), see [Install Docker Engine](https://docs.docker.com/engine/install/).
@@ -44,7 +44,7 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
 
    `vm.max_map_count`. This value sets the maximum number of memory map areas a process may have. Its default value is 65530. While most applications require fewer than a thousand maps, reducing this value can result in abnormal behaviors, and the system will throw out-of-memory errors when a process reaches the limitation.
 
-   RAGFlow v0.19.0 uses Elasticsearch or [Infinity](https://github.com/infiniflow/infinity) for multiple recall. Setting the value of `vm.max_map_count` correctly is crucial to the proper functioning of the Elasticsearch component.
+   RAGFlow v0.19.1 uses Elasticsearch or [Infinity](https://github.com/infiniflow/infinity) for multiple recall. Setting the value of `vm.max_map_count` correctly is crucial to the proper functioning of the Elasticsearch component.
 
 <Tabs
   defaultValue="linux"
@@ -184,13 +184,13 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
    ```bash
    $ git clone https://github.com/infiniflow/ragflow.git
    $ cd ragflow/docker
-   $ git checkout -f v0.19.0
+   $ git checkout -f v0.19.1
    ```
 
 3. Use the pre-built Docker images and start up the server:
 
    :::tip NOTE
-   The command below downloads the `v0.19.0-slim` edition of the RAGFlow Docker image. Refer to the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.19.0-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0` for the full edition `v0.19.0`.
+   The command below downloads the `v0.19.1-slim` edition of the RAGFlow Docker image. Refer to the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.19.1-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` for the full edition `v0.19.1`.
    :::
 
    ```bash
@@ -205,10 +205,10 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
 <APITable>
 ```
 
-| RAGFlow image tag   | Image size (GB) | Has embedding models and Python packages?:collision: | Stable?                  |
+| RAGFlow image tag   | Image size (GB) | Has embedding models and Python packages? | Stable?                  |
 | ------------------- | --------------- | ----------------------------------------- | ------------------------ |
-| `v0.19.0`           | &approx;9       | :heavy_check_mark:                        | Stable release           |
-| `v0.19.0-slim`      | &approx;2       | ❌                                        | Stable release           |
+| `v0.19.1`           | &approx;9       | :heavy_check_mark:                        | Stable release           |
+| `v0.19.1-slim`      | &approx;2       | ❌                                        | Stable release           |
 | `nightly`           | &approx;9       | :heavy_check_mark:                        | *Unstable* nightly build |
 | `nightly-slim`      | &approx;2       | ❌                                        | *Unstable* nightly build |
 
@@ -217,12 +217,12 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
 ```
 
 :::danger IMPORTANT
-:collision: The embedding models included in `v0.19.0` and `nightly` are:
+The embedding models included in `v0.19.1` and `nightly` are:
 
 - BAAI/bge-large-zh-v1.5
 - maidalun1020/bce-embedding-base_v1
 
-Please note these two embedding models support both English and Chinese. If your knowledge base contains other languages, the performance may be COMPROMISED.
+These two embedding models are optimized specifically for English and Chinese, so performance may be compromised if you use them to embed documents in other languages.
 :::
 
 4. Check the server status after having the server up and running:
@@ -287,7 +287,7 @@ To add and configure an LLM:
 
 ## Create your first knowledge base
 
-You are allowed to upload files to a knowledge base in RAGFlow and parse them into datasets. A knowledge base is virtually a collection of datasets. Question answering in RAGFlow can be based on a particular knowledge base or multiple knowledge bases. File formats that RAGFlow supports include documents (PDF, DOC, DOCX, TXT, MD), tables (CSV, XLSX, XLS), pictures (JPEG, JPG, PNG, TIF, GIF), and slides (PPT, PPTX).
+You are allowed to upload files to a knowledge base in RAGFlow and parse them into datasets. A knowledge base is virtually a collection of datasets. Question answering in RAGFlow can be based on a particular knowledge base or multiple knowledge bases. File formats that RAGFlow supports include documents (PDF, DOC, DOCX, TXT, MD, MDX), tables (CSV, XLSX, XLS), pictures (JPEG, JPG, PNG, TIF, GIF), and slides (PPT, PPTX).
 
 To create your first knowledge base:
 

docs/references/glossary.mdx

@@ -19,7 +19,7 @@ import TOCInline from '@theme/TOCInline';
 
 ### Cross-language search
 
-Cross-language search (also known as cross-lingual retrieval) is a feature introduced in version 0.19.0. It enables users to submit queries in one language (for example, English) and retrieve relevant documents written in other languages such as Chinese or Spanish. This feature is enabled by the system’s default chat model, which translates queries to ensure accurate matching of semantic meaning across languages.
+Cross-language search (also known as cross-lingual retrieval) is a feature introduced in version 0.19.1. It enables users to submit queries in one language (for example, English) and retrieve relevant documents written in other languages such as Chinese or Spanish. This feature is enabled by the system’s default chat model, which translates queries to ensure accurate matching of semantic meaning across languages.
 
 By enabling cross-language search, users can effortlessly access a broader range of information regardless of language barriers, significantly enhancing the system’s usability and inclusiveness.
 

docs/references/http_api_reference.md

@@ -343,7 +343,6 @@ Creates a dataset.
   - `"embedding_model"`: `string`
   - `"permission"`: `string`
   - `"chunk_method"`: `string`
-  - `"pagerank"`: `int`
   - `"parser_config"`: `object`
 
 ##### Request example
@@ -384,12 +383,6 @@ curl --request POST \
   - `"me"`: (Default) Only you can manage the dataset.
   - `"team"`: All team members can manage the dataset.
 
-- `"pagerank"`: (*Body parameter*), `int`  
-  refer to [Set page rank](https://ragflow.io/docs/dev/set_page_rank)
-  - Default: `0`
-  - Minimum: `0`
-  - Maximum: `100`
-
 - `"chunk_method"`: (*Body parameter*), `enum<string>`  
   The chunking method of the dataset to create. Available options:  
   - `"naive"`: General (default)
@@ -900,7 +893,7 @@ curl --request PUT \
 - `document_id`: (*Path parameter*)  
   The ID of the document to update.
 - `"name"`: (*Body parameter*), `string`
-- `"meta_fields"`: (*Body parameter*)， `dict[str, Any]` The meta fields of the document.
+- `"meta_fields"`: (*Body parameter*), `dict[str, Any]` The meta fields of the document.
 - `"chunk_method"`: (*Body parameter*), `string`  
   The parsing method to apply to the document:  
   - `"naive"`: General
@@ -2142,7 +2135,7 @@ Success:
         "id": "4606b4ec87ad11efbc4f0242ac120006",
         "messages": [
             {
-                "content": "Hi! I am your assistant，can I help you?",
+                "content": "Hi! I am your assistant, can I help you?",
                 "role": "assistant"
             }
         ],
@@ -2283,7 +2276,7 @@ Success:
             "id": "578d541e87ad11ef96b90242ac120006",
             "messages": [
                 {
-                    "content": "Hi! I am your assistant，can I help you?",
+                    "content": "Hi! I am your assistant, can I help you?",
                     "role": "assistant"
                 }
             ],
@@ -3227,22 +3220,22 @@ Failure:
 
 ---
 
-### Related Questions
+### Generate related questions
 
-**POST** `/v1/conversation/related_questions`
+**POST** `/v1/sessions/related_questions`
 
 Generates five to ten alternative question strings from the user's original query to retrieve more relevant search results.
 
-This operation requires a `Bearer Login Token`, typically expires with in 24 hours. You can find the it in the browser request easily.
+This operation requires a `Bearer Login Token`, which typically expires with in 24 hours. You can find the it in the Request Headers in your browser easily.
 
 :::tip NOTE
-The chat model dynamically determines the number of questions to generate based on the instruction, typically between five and ten.
+The chat model autonomously determines the number of questions to generate based on the instruction, typically between five and ten.
 :::
 
 #### Request
 
 - Method: POST
-- URL: `/v1/conversation/related_questions`
+- URL: `/v1/sessions/related_questions`
 - Headers:
   - `'content-Type: application/json'`
   - `'Authorization: Bearer <YOUR_LOGIN_TOKEN>'`
@@ -3253,7 +3246,7 @@ The chat model dynamically determines the number of questions to generate based
 
 ```bash
 curl --request POST \
-     --url http://{address}/v1/conversation/related_questions \
+     --url http://{address}/v1/sessions/related_questions \
      --header 'Content-Type: application/json' \
      --header 'Authorization: Bearer <YOUR_LOGIN_TOKEN>' \
      --data '

docs/references/python_api_reference.md

@@ -100,7 +100,6 @@ RAGFlow.create_dataset(
     embedding_model: Optional[str] = "BAAI/bge-large-zh-v1.5@BAAI",
     permission: str = "me", 
     chunk_method: str = "naive",
-    pagerank: int = 0,
     parser_config: DataSet.ParserConfig = None
 ) -> DataSet
 ```
@@ -148,10 +147,6 @@ The chunking method of the dataset to create. Available options:
 - `"one"`: One
 - `"email"`: Email
 
-##### pagerank, `int`
-
-The pagerank of the dataset to create. Defaults to `0`.
-
 ##### parser_config
 
 The parser configuration of the dataset. A `ParserConfig` object's attributes vary based on the selected `chunk_method`:
@@ -1238,7 +1233,7 @@ The name of the chat session to create.
 - Success: A `Session` object containing the following attributes:
   - `id`: `str` The auto-generated unique identifier of the created session.
   - `name`: `str` The name of the created session.
-  - `message`: `list[Message]` The opening message of the created session. Default: `[{"role": "assistant", "content": "Hi! I am your assistant，can I help you?"}]`
+  - `message`: `list[Message]` The opening message of the created session. Default: `[{"role": "assistant", "content": "Hi! I am your assistant, can I help you?"}]`
   - `chat_id`: `str` The ID of the associated chat assistant.
 - Failure: `Exception`
 
@@ -1497,7 +1492,7 @@ The parameters in `begin` component.
 
 - Success: A `Session` object containing the following attributes:
   - `id`: `str` The auto-generated unique identifier of the created session.
-  - `message`: `list[Message]` The messages of the created session assistant. Default: `[{"role": "assistant", "content": "Hi! I am your assistant，can I help you?"}]`
+  - `message`: `list[Message]` The messages of the created session assistant. Default: `[{"role": "assistant", "content": "Hi! I am your assistant, can I help you?"}]`
   - `agent_id`: `str` The ID of the associated agent.
 - Failure: `Exception`
 

docs/release_notes.md

@@ -14,12 +14,12 @@ Each RAGFlow release is available in two editions:
 :::
 
 :::danger IMPORTANT
-:collision: The embedding models included in a full edition are:
+The embedding models included in a full edition are:
 
 - BAAI/bge-large-zh-v1.5
 - maidalun1020/bce-embedding-base_v1
 
-Please note these two embedding models support both English and Chinese. If your knowledge base contains other languages, the performance may be COMPROMISED.
+These two embedding models are optimized specifically for English and Chinese, so performance may be compromised if you use them to embed documents in other languages.
 :::
 
 ## v0.19.0
@@ -28,16 +28,24 @@ Released on May 26, 2025.
 
 ### New features
 
-- Cross-language search is supported in the Knowledge and Chat modules, enhancing search accuracy and user experience in multilingual environments, such as in Chinese-English knowledge bases.
+- [Cross-language search](./references/glossary.mdx#cross-language-search) is supported in the Knowledge and Chat modules, enhancing search accuracy and user experience in multilingual environments, such as in Chinese-English knowledge bases.
 - Agent component: A new Code component supports Python and JavaScript scripts, enabling developers to handle more complex tasks like dynamic data processing.
 - Enhanced image display: Images in Chat and Search now render directly within responses, rather than as external references. Knowledge retrieval testing can retrieve images directly, instead of texts extracted from images.
-- Claude 4: Developers can now use the newly released, most advanced Claude model.
+- Claude 4 and ChatGPT o3: Developers can now use the newly released, most advanced Claude model and OpenAI’s latest ChatGPT o3 inference model.
 
 > The following features are contributed by our community contributors:
 
-- Agent component: Enables tool calling within the Generate Component. Kudos to [notsyncing](https://github.com/notsyncing).
-- Markdown rendering: Image references in a markdown file can be displayed after chunking. Kudos to [Woody-Hu](https://github.com/Woody-Hu).
-- Vector database support: OpenSearch can now be used as RAGFlow's document engine. Kudos to [pyyuhao](https://github.com/pyyuhao).
+- Agent component: Enables tool calling within the Generate Component. Thanks to [notsyncing](https://github.com/notsyncing).
+- Markdown rendering: Image references in a markdown file can be displayed after chunking. Thanks to [Woody-Hu](https://github.com/Woody-Hu).
+- Document engine support: OpenSearch can now be used as RAGFlow's document engine. Thanks to [pyyuhao](https://github.com/pyyuhao).
+
+### Documentation
+
+#### Added documents
+
+- [Select PDF parser](./guides/dataset/select_pdf_parser.md)
+- [Enable Excel2HTML](./guides/dataset/enable_excel2html.md)
+- [Code component](./guides/agent/agent_component_reference/code.mdx)
 
 ## v0.18.0
 

download_deps.py

@@ -11,12 +11,13 @@
 # ///
 
 from huggingface_hub import snapshot_download
+from typing import Union
 import nltk
 import os
 import urllib.request
 import argparse
 
-def get_urls(use_china_mirrors=False):
+def get_urls(use_china_mirrors=False) -> Union[str, list[str]]:
     if use_china_mirrors:
         return [
             "http://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb",
@@ -24,8 +25,8 @@ def get_urls(use_china_mirrors=False):
             "https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar",
             "https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar.md5",
             "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
-            "https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chrome-linux64.zip",
-            "https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip",
+            ["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"],
+            ["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"],
         ]
     else:
         return [
@@ -49,7 +50,7 @@ repos = [
 def download_model(repo_id):
     local_dir = os.path.abspath(os.path.join("huggingface.co", repo_id))
     os.makedirs(local_dir, exist_ok=True)
-    snapshot_download(repo_id=repo_id, local_dir=local_dir, local_dir_use_symlinks=False)
+    snapshot_download(repo_id=repo_id, local_dir=local_dir)
 
 
 if __name__ == "__main__":
@@ -60,10 +61,11 @@ if __name__ == "__main__":
     urls = get_urls(args.china_mirrors)
     
     for url in urls:
-        filename = url.split("/")[-1]
-        print(f"Downloading {url}...")
+        download_url = url[0] if isinstance(url, list) else url
+        filename = url[1] if isinstance(url, list) else url.split("/")[-1]
+        print(f"Downloading {filename} from {download_url}...")
         if not os.path.exists(filename):
-            urllib.request.urlretrieve(url, filename)
+            urllib.request.urlretrieve(download_url, filename)
 
     local_dir = os.path.abspath('nltk_data')
     for data in ['wordnet', 'punkt', 'punkt_tab']:
@@ -72,4 +74,4 @@ if __name__ == "__main__":
 
     for repo_id in repos:
         print(f"Downloading huggingface repo {repo_id}...")
-        download_model(repo_id)
+        download_model(repo_id)

graphrag/entity_resolution.py

@@ -94,25 +94,52 @@ class EntityResolution(Extractor):
             candidate_resolution[k] = [(a, b) for a, b in itertools.combinations(v, 2) if (a in subgraph_nodes or b in subgraph_nodes) and self.is_similarity(a, b)]
         num_candidates = sum([len(candidates) for _, candidates in candidate_resolution.items()])
         callback(msg=f"Identified {num_candidates} candidate pairs")
+        remain_candidates_to_resolve = num_candidates
 
         resolution_result = set()
+        resolution_result_lock = trio.Lock()
         resolution_batch_size = 100
+        max_concurrent_tasks = 5
+        semaphore = trio.Semaphore(max_concurrent_tasks)
+
+        async def limited_resolve_candidate(candidate_batch, result_set, result_lock):
+            nonlocal remain_candidates_to_resolve, callback
+            async with semaphore:
+                try:
+                    with trio.move_on_after(180) as cancel_scope:
+                        await self._resolve_candidate(candidate_batch, result_set, result_lock)
+                        remain_candidates_to_resolve = remain_candidates_to_resolve - len(candidate_batch[1])
+                        callback(msg=f"Resolved {len(candidate_batch[1])} pairs, {remain_candidates_to_resolve} are remained to resolve. ")
+                    if cancel_scope.cancelled_caught:
+                        logging.warning(f"Timeout resolving {candidate_batch}, skipping...")
+                        remain_candidates_to_resolve = remain_candidates_to_resolve - len(candidate_batch[1])
+                        callback(msg=f"Fail to resolved {len(candidate_batch[1])} pairs due to timeout reason, skipped. {remain_candidates_to_resolve} are remained to resolve. ")
+                except Exception as e:
+                    logging.error(f"Error resolving candidate batch: {e}")
+
+
         async with trio.open_nursery() as nursery:
             for candidate_resolution_i in candidate_resolution.items():
                 if not candidate_resolution_i[1]:
                     continue
                 for i in range(0, len(candidate_resolution_i[1]), resolution_batch_size):
                     candidate_batch = candidate_resolution_i[0], candidate_resolution_i[1][i:i + resolution_batch_size]
-                    nursery.start_soon(self._resolve_candidate, candidate_batch, resolution_result)
+                    nursery.start_soon(limited_resolve_candidate, candidate_batch, resolution_result, resolution_result_lock)
+
         callback(msg=f"Resolved {num_candidates} candidate pairs, {len(resolution_result)} of them are selected to merge.")
 
         change = GraphChange()
         connect_graph = nx.Graph()
         connect_graph.add_edges_from(resolution_result)
+
+        async def limited_merge_nodes(graph, nodes, change):
+            async with semaphore:
+                await self._merge_graph_nodes(graph, nodes, change)
+
         async with trio.open_nursery() as nursery:
             for sub_connect_graph in nx.connected_components(connect_graph):
                 merging_nodes = list(sub_connect_graph)
-                nursery.start_soon(self._merge_graph_nodes, graph, merging_nodes, change)
+                nursery.start_soon(limited_merge_nodes, graph, merging_nodes, change)
 
         # Update pagerank
         pr = nx.pagerank(graph)
@@ -124,7 +151,7 @@ class EntityResolution(Extractor):
             change=change,
         )
 
-    async def _resolve_candidate(self, candidate_resolution_i: tuple[str, list[tuple[str, str]]], resolution_result: set[str]):
+    async def _resolve_candidate(self, candidate_resolution_i: tuple[str, list[tuple[str, str]]], resolution_result: set[str], resolution_result_lock: trio.Lock):
         gen_conf = {"temperature": 0.5}
         pair_txt = [
             f'When determining whether two {candidate_resolution_i[0]}s are the same, you should only focus on critical properties and overlook noisy factors.\n']
@@ -142,7 +169,16 @@ class EntityResolution(Extractor):
         text = perform_variable_replacements(self._resolution_prompt, variables=variables)
         logging.info(f"Created resolution prompt {len(text)} bytes for {len(candidate_resolution_i[1])} entity pairs of type {candidate_resolution_i[0]}")
         async with chat_limiter:
-            response = await trio.to_thread.run_sync(lambda: self._chat(text, [{"role": "user", "content": "Output:"}], gen_conf))
+            try:
+                with trio.move_on_after(120) as cancel_scope:
+                    response = await trio.to_thread.run_sync(self._chat, text, [{"role": "user", "content": "Output:"}], gen_conf)
+                if cancel_scope.cancelled_caught:
+                    logging.warning("_resolve_candidate._chat timeout, skipping...")
+                    return
+            except Exception as e:
+                logging.error(f"_resolve_candidate._chat failed: {e}")
+                return
+
         logging.debug(f"_resolve_candidate chat prompt: {text}\nchat response: {response}")
         result = self._process_results(len(candidate_resolution_i[1]), response,
                                        self.prompt_variables.get(self._record_delimiter_key,
@@ -151,8 +187,9 @@ class EntityResolution(Extractor):
                                                             DEFAULT_ENTITY_INDEX_DELIMITER),
                                        self.prompt_variables.get(self._resolution_result_delimiter_key,
                                                             DEFAULT_RESOLUTION_RESULT_DELIMITER))
-        for result_i in result:
-            resolution_result.add(candidate_resolution_i[1][result_i[0] - 1])
+        async with resolution_result_lock:
+            for result_i in result:
+                resolution_result.add(candidate_resolution_i[1][result_i[0] - 1])
 
     def _process_results(
             self,
@@ -185,6 +222,7 @@ class EntityResolution(Extractor):
         if is_english(a) and is_english(b):
             if editdistance.eval(a, b) <= min(len(a), len(b)) // 2:
                 return True
+            return False
 
         if len(set(a) & set(b)) > 1:
             return True

graphrag/general/community_reports_extractor.py

@@ -89,7 +89,15 @@ class CommunityReportsExtractor(Extractor):
             text = perform_variable_replacements(self._extraction_prompt, variables=prompt_variables)
             gen_conf = {"temperature": 0.3}
             async with chat_limiter:
-                response = await trio.to_thread.run_sync(lambda: self._chat(text, [{"role": "user", "content": "Output:"}], gen_conf))
+                try:
+                    with trio.move_on_after(120) as cancel_scope:
+                        response = await trio.to_thread.run_sync( self._chat, text, [{"role": "user", "content": "Output:"}], gen_conf)
+                    if cancel_scope.cancelled_caught:
+                        logging.warning("extract_community_report._chat timeout, skipping...")
+                        return
+                except Exception as e:
+                    logging.error(f"extract_community_report._chat failed: {e}")
+                    return
             token_count += num_tokens_from_string(text + response)
             response = re.sub(r"^[^\{]*", "", response)
             response = re.sub(r"[^\}]*$", "", response)