fix ollama bug (#999)

### What problem does this PR solve?


### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

Dockerfile.arm

@@ -0,0 +1,32 @@
+FROM python:3.11
+USER  root
+
+WORKDIR /ragflow
+
+COPY requirements_arm.txt /ragflow/requirements.txt
+RUN pip install -i https://mirrors.aliyun.com/pypi/simple/ --default-timeout=1000 -r requirements.txt &&\
+python -c "import nltk;nltk.download('punkt');nltk.download('wordnet')"
+
+RUN apt-get update && \
+    apt-get install -y curl gnupg && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - && \
+    apt-get install -y nodejs nginx ffmpeg libsm6 libxext6 libgl1
+
+ADD ./web ./web
+RUN cd ./web && npm i --force && npm run build
+
+ADD ./api ./api
+ADD ./conf ./conf
+ADD ./deepdoc ./deepdoc
+ADD ./rag ./rag
+
+ENV PYTHONPATH=/ragflow/
+ENV HF_ENDPOINT=https://hf-mirror.com
+
+ADD docker/entrypoint.sh ./entrypoint.sh
+ADD docker/.env ./
+RUN chmod +x ./entrypoint.sh
+
+ENTRYPOINT ["./entrypoint.sh"]

Dockerfile.cuda

@@ -1,4 +1,4 @@
-FROM FROM infiniflow/ragflow-base:v2.0
+FROM infiniflow/ragflow-base:v2.0
 USER  root
 
 WORKDIR /ragflow

README.md

@@ -17,28 +17,53 @@
     <a href="https://demo.ragflow.io" target="_blank">
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"></a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.6.0-brightgreen"
-            alt="docker pull infiniflow/ragflow:v0.6.0"></a>
-      <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
-    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=1570EF" alt="license">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.7.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.7.0"></a>
+    <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
+    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=2e6cc4" alt="license">
   </a>
 </p>
 
+<details open>
+<summary></b>📕 Table of Contents</b></summary>
+  
+- 💡 [What is RAGFlow?](#-what-is-ragflow)
+- 🎮 [Demo](#-demo)
+- 📌 [Latest Updates](#-latest-updates)
+- 🌟 [Key Features](#-key-features)
+- 🔎 [System Architecture](#-system-architecture)
+- 🎬 [Get Started](#-get-started)
+- 🔧 [Configurations](#-configurations)
+- 🛠️ [Build from source](#-build-from-source)
+- 🛠️ [Launch service from source](#-launch-service-from-source)
+- 📚 [Documentation](#-documentation)
+- 📜 [Roadmap](#-roadmap)
+- 🏄 [Community](#-community)
+- 🙌 [Contributing](#-contributing)
+
+</details>
+
 ## 💡 What is RAGFlow?
 
 [RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It offers a streamlined RAG workflow for businesses of any scale, combining LLM (Large Language Models) to provide truthful question-answering capabilities, backed by well-founded citations from various complex formatted data.
 
+## 🎮 Demo
+
+Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
+
 ## 📌 Latest Updates
 
+- 2024-05-30 Integrates [BCE](https://github.com/netease-youdao/BCEmbedding), [BGE](https://github.com/FlagOpen/FlagEmbedding), and [Colbert](https://github.com/stanford-futuredata/ColBERT) reranker models.
+- 2024-05-28 Supports LLM Baichuan and VolcanoArk.
+- 2024-05-23 Supports [RAPTOR](https://arxiv.org/html/2401.18059v1) for better text retrieval.
 - 2024-05-21 Supports streaming output and text chunk retrieval API. 
 - 2024-05-15 Integrates OpenAI GPT-4o.
 - 2024-05-08 Integrates LLM DeepSeek-V2.
 - 2024-04-26 Adds file management.
-- 2024-04-19 Supports conversation API ([detail](./docs/conversation_api.md)).
+- 2024-04-19 Supports conversation API ([detail](./docs/references/api.md)).
 - 2024-04-16 Integrates an embedding model 'bce-embedding-base_v1' from [BCEmbedding](https://github.com/netease-youdao/BCEmbedding), and [FastEmbed](https://github.com/qdrant/fastembed), which is designed specifically for light and speedy embedding.
-- 2024-04-11 Supports [Xinference](./docs/xinference.md) for local LLM deployment.
+- 2024-04-11 Supports [Xinference](./docs/guides/deploy_local_llm.md) for local LLM deployment.
 - 2024-04-10 Adds a new layout recognition model for analyzing legal documents.
-- 2024-04-08 Supports [Ollama](./docs/ollama.md) for local LLM deployment.
+- 2024-04-08 Supports [Ollama](./docs/guides/deploy_local_llm.md) for local LLM deployment.
 - 2024-04-07 Supports Chinese UI.
 
 ## 🌟 Key Features
@@ -87,7 +112,7 @@
 
 ### 🚀 Start up the server
 
-1. Ensure `vm.max_map_count` >= 262144 ([more](./docs/max_map_count.md)):
+1. Ensure `vm.max_map_count` >= 262144 ([more](./docs/guides/max_map_count.md)):
 
    > To check the value of `vm.max_map_count`:
    >
@@ -116,7 +141,7 @@
 
 3. Build the pre-built Docker images and start up the server:
 
-   > Running the following commands automatically downloads the *dev* version RAGFlow Docker image. To download and run a specified Docker version, update `RAGFLOW_VERSION` in **docker/.env** to the intended version, for example `RAGFLOW_VERSION=v0.6.0`, before running the following commands.
+   > Running the following commands automatically downloads the *dev* version RAGFlow Docker image. To download and run a specified Docker version, update `RAGFLOW_VERSION` in **docker/.env** to the intended version, for example `RAGFLOW_VERSION=v0.7.0`, before running the following commands.
 
    ```bash
    $ cd ragflow/docker
@@ -154,7 +179,7 @@
    > With default settings, you only need to enter `http://IP_OF_YOUR_MACHINE` (**sans** port number) as the default HTTP serving port `80` can be omitted when using the default configurations.
 6. In [service_conf.yaml](./docker/service_conf.yaml), select the desired LLM factory in `user_default_llm` and update the `API_KEY` field with the corresponding API key.
 
-   > See [./docs/llm_api_key_setup.md](./docs/llm_api_key_setup.md) for more information.
+   > See [./docs/guides/llm_api_key_setup.md](./docs/guides/llm_api_key_setup.md) for more information.
 
    _The show is now on!_
 
@@ -191,93 +216,104 @@ $ chmod +x ./entrypoint.sh
 $ docker compose up -d
 ```
 
-## 🛠️ Launch Service from Source
+## 🛠️ Launch service from source
 
-To launch the service from source, please follow these steps:
+To launch the service from source:
 
-1. Clone the repository
-```bash
-$ git clone https://github.com/infiniflow/ragflow.git
-$ cd ragflow/
-```
+1. Clone the repository: 
 
-2. Create a virtual environment (ensure Anaconda or Miniconda is installed)
-```bash
-$ conda create -n ragflow python=3.11.0
-$ conda activate ragflow
-$ pip install -r requirements.txt
-```
-If CUDA version is greater than 12.0, execute the following additional commands:
-```bash
-$ pip uninstall -y onnxruntime-gpu
-$ pip install onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
-```
+   ```bash
+   $ git clone https://github.com/infiniflow/ragflow.git
+   $ cd ragflow/
+   ```
 
-3. Copy the entry script and configure environment variables
-```bash
-$ cp docker/entrypoint.sh .
-$ vi entrypoint.sh
-```
-Use the following commands to obtain the Python path and the ragflow project path:
-```bash
-$ which python
-$ pwd
-```
+2. Create a virtual environment, ensuring that Anaconda or Miniconda is installed:
 
-Set the output of `which python` as the value for `PY` and the output of `pwd` as the value for `PYTHONPATH`.
+   ```bash
+   $ conda create -n ragflow python=3.11.0
+   $ conda activate ragflow
+   $ pip install -r requirements.txt
+   ```
+   
+   ```bash
+   # If your CUDA version is higher than 12.0, run the following additional commands:
+   $ pip uninstall -y onnxruntime-gpu
+   $ pip install onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
+   ```
 
-If `LD_LIBRARY_PATH` is already configured, it can be commented out.
+3. Copy the entry script and configure environment variables:
 
-```bash
-# Adjust configurations according to your actual situation; the two export commands are newly added.
-PY=${PY}
-export PYTHONPATH=${PYTHONPATH}
-# Optional: Add Hugging Face mirror
-export HF_ENDPOINT=https://hf-mirror.com
-```
+   ```bash
+   # Get the Python path:
+   $ which python
+   # Get the ragflow project path:
+   $ pwd
+   ```
+   
+   ```bash
+   $ cp docker/entrypoint.sh .
+   $ vi entrypoint.sh
+   ```
 
-4. Start the base services
-```bash
-$ cd docker
-$ docker compose -f docker-compose-base.yml up -d 
-```
+   ```bash
+   # Adjust configurations according to your actual situation (the following two export commands are newly added):
+   # - Assign the result of `which python` to `PY`.
+   # - Assign the result of `pwd` to `PYTHONPATH`.
+   # - Comment out `LD_LIBRARY_PATH`, if it is configured.
+   # - Optional: Add Hugging Face mirror.
+   PY=${PY}
+   export PYTHONPATH=${PYTHONPATH}
+   export HF_ENDPOINT=https://hf-mirror.com
+   ```
 
-5. Check the configuration files
-Ensure that the settings in **docker/.env** match those in **conf/service_conf.yaml**. The IP addresses and ports for related services in **service_conf.yaml** should be changed to the local machine IP and ports exposed by the container.
+4. Launch the third-party services (MinIO, Elasticsearch, Redis, and MySQL):
 
-6. Launch the service
-```bash
-$ chmod +x ./entrypoint.sh
-$ bash ./entrypoint.sh
-```
+   ```bash
+   $ cd docker
+   $ docker compose -f docker-compose-base.yml up -d 
+   ```
 
-7. Start the WebUI service
-```bash
-$ cd web
-$ npm install --registry=https://registry.npmmirror.com --force
-$ vim .umirc.ts
-# Modify proxy.target to 127.0.0.1:9380
-$ npm run dev 
-```
+5. Check the configuration files, ensuring that:
 
-8. Deploy the WebUI service
-```bash
-$ cd web
-$ npm install --registry=https://registry.npmmirror.com --force
-$ umi build
-$ mkdir -p /ragflow/web
-$ cp -r dist /ragflow/web
-$ apt install nginx -y
-$ cp ../docker/nginx/proxy.conf /etc/nginx
-$ cp ../docker/nginx/nginx.conf /etc/nginx
-$ cp ../docker/nginx/ragflow.conf /etc/nginx/conf.d
-$ systemctl start nginx
-```
+   - The settings in **docker/.env** match those in **conf/service_conf.yaml**. 
+   - The IP addresses and ports for related services in **service_conf.yaml** match the local machine IP and ports exposed by the container.
+
+6. Launch the RAGFlow backend service:
+
+   ```bash
+   $ chmod +x ./entrypoint.sh
+   $ bash ./entrypoint.sh
+   ```
+
+7. Launch the frontend service:
+
+   ```bash
+   $ cd web
+   $ npm install --registry=https://registry.npmmirror.com --force
+   $ vim .umirc.ts
+   # Update proxy.target to 127.0.0.1:9380
+   $ npm run dev 
+   ```
+
+8. Deploy the frontend service:
+
+   ```bash
+   $ cd web
+   $ npm install --registry=https://registry.npmmirror.com --force
+   $ umi build
+   $ mkdir -p /ragflow/web
+   $ cp -r dist /ragflow/web
+   $ apt install nginx -y
+   $ cp ../docker/nginx/proxy.conf /etc/nginx
+   $ cp ../docker/nginx/nginx.conf /etc/nginx
+   $ cp ../docker/nginx/ragflow.conf /etc/nginx/conf.d
+   $ systemctl start nginx
+   ```
 
 ## 📚 Documentation
 
 - [Quickstart](./docs/quickstart.md)
-- [FAQ](./docs/faq.md)
+- [FAQ](./docs/references/faq.md)
 
 ## 📜 Roadmap
 
@@ -290,4 +326,4 @@ See the [RAGFlow Roadmap 2024](https://github.com/infiniflow/ragflow/issues/162)
 
 ## 🙌 Contributing
 
-RAGFlow flourishes via open-source collaboration. In this spirit, we embrace diverse contributions from the community. If you would like to be a part, review our [Contribution Guidelines](https://github.com/infiniflow/ragflow/blob/main/docs/CONTRIBUTING.md) first.
+RAGFlow flourishes via open-source collaboration. In this spirit, we embrace diverse contributions from the community. If you would like to be a part, review our [Contribution Guidelines](./docs/references/CONTRIBUTING.md) first.

README_ja.md

@@ -17,10 +17,10 @@
     <a href="https://demo.ragflow.io" target="_blank">
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"></a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.6.0-brightgreen"
-            alt="docker pull infiniflow/ragflow:v0.6.0"></a>
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.7.0-brightgreen"
+            alt="docker pull infiniflow/ragflow:v0.7.0"></a>
       <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
-    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=1570EF" alt="license">
+    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=2e6cc4" alt="license">
   </a>
 </p>
 
@@ -28,18 +28,25 @@
 
 [RAGFlow](https://ragflow.io/) は、深い文書理解に基づいたオープンソースの RAG (Retrieval-Augmented Generation) エンジンである。LLM（大規模言語モデル）を組み合わせることで、様々な複雑なフォーマットのデータから根拠のある引用に裏打ちされた、信頼できる質問応答機能を実現し、あらゆる規模のビジネスに適した RAG ワークフローを提供します。
 
+## 🎮 Demo
+
+デモをお試しください：[https://demo.ragflow.io](https://demo.ragflow.io)。
+
 ## 📌 最新情報
 
+- 2024-05-30 [BCE](https://github.com/netease-youdao/BCEmbedding)、[BGE](https://github.com/FlagOpen/FlagEmbedding)、[Colbert](https://github.com/stanford-futuredata/ColBERT) reranker を統合。
+- 2024-05-28 LLM BaichuanとVolcanoArkを統合しました。
+- 2024-05-23 より良いテキスト検索のために[RAPTOR](https://arxiv.org/html/2401.18059v1)をサポート。
 - 2024-05-21 ストリーミング出力とテキストチャンク取得APIをサポート。
 - 2024-05-15 OpenAI GPT-4oを統合しました。
 - 2024-05-08 LLM DeepSeek-V2を統合しました。
 - 2024-04-26 「ファイル管理」機能を追加しました。
-- 2024-04-19 会話 API をサポートします ([詳細](./docs/conversation_api.md))。
+- 2024-04-19 会話 API をサポートします ([詳細](./docs/references/api.md))。
 - 2024-04-16 [BCEmbedding](https://github.com/netease-youdao/BCEmbedding) から埋め込みモデル「bce-embedding-base_v1」を追加します。
 - 2024-04-16 [FastEmbed](https://github.com/qdrant/fastembed) は、軽量かつ高速な埋め込み用に設計されています。
-- 2024-04-11 ローカル LLM デプロイメント用に [Xinference](./docs/xinference.md) をサポートします。
+- 2024-04-11 ローカル LLM デプロイメント用に [Xinference](./docs/guides/deploy_local_llm.md) をサポートします。
 - 2024-04-10 メソッド「Laws」に新しいレイアウト認識モデルを追加します。
-- 2024-04-08 [Ollama](./docs/ollama.md) を使用した大規模モデルのローカライズされたデプロイメントをサポートします。
+- 2024-04-08 [Ollama](./docs/guides/deploy_local_llm.md) を使用した大規模モデルのローカライズされたデプロイメントをサポートします。
 - 2024-04-07 中国語インターフェースをサポートします。
 
 
@@ -89,7 +96,7 @@
 
 ### 🚀 サーバーを起動
 
-1. `vm.max_map_count` >= 262144 であることを確認する【[もっと](./docs/max_map_count.md)】:
+1. `vm.max_map_count` >= 262144 であることを確認する【[もっと](./docs/guides/max_map_count.md)】:
 
    > `vm.max_map_count` の値をチェックするには:
    >
@@ -124,7 +131,7 @@
    $ docker compose up -d
    ```
 
-   > 上記のコマンドを実行すると、RAGFlowの開発版dockerイメージが自動的にダウンロードされます。 特定のバージョンのDockerイメージをダウンロードして実行したい場合は、docker/.envファイルのRAGFLOW_VERSION変数を見つけて、対応するバージョンに変更してください。 例えば、RAGFLOW_VERSION=v0.6.0として、上記のコマンドを実行してください。
+   > 上記のコマンドを実行すると、RAGFlowの開発版dockerイメージが自動的にダウンロードされます。 特定のバージョンのDockerイメージをダウンロードして実行したい場合は、docker/.envファイルのRAGFLOW_VERSION変数を見つけて、対応するバージョンに変更してください。 例えば、RAGFLOW_VERSION=v0.7.0として、上記のコマンドを実行してください。
 
    > コアイメージのサイズは約 9 GB で、ロードに時間がかかる場合があります。
 
@@ -155,7 +162,7 @@
    > デフォルトの設定を使用する場合、デフォルトの HTTP サービングポート `80` は省略できるので、与えられたシナリオでは、`http://IP_OF_YOUR_MACHINE`（ポート番号は省略）だけを入力すればよい。
 6. [service_conf.yaml](./docker/service_conf.yaml) で、`user_default_llm` で希望の LLM ファクトリを選択し、`API_KEY` フィールドを対応する API キーで更新する。
 
-   > 詳しくは [./docs/llm_api_key_setup.md](./docs/llm_api_key_setup.md) を参照してください。
+   > 詳しくは [./docs/guides/llm_api_key_setup.md](./docs/guides/llm_api_key_setup.md) を参照してください。
 
    _これで初期設定完了！ショーの開幕です！_
 
@@ -186,7 +193,7 @@
 ```bash
 $ git clone https://github.com/infiniflow/ragflow.git
 $ cd ragflow/
-$ docker build -t infiniflow/ragflow:v0.6.0 .
+$ docker build -t infiniflow/ragflow:v0.7.0 .
 $ cd ragflow/docker
 $ chmod +x ./entrypoint.sh
 $ docker compose up -d
@@ -255,7 +262,7 @@ $ bash ./entrypoint.sh
 ## 📚 ドキュメンテーション
 
 - [Quickstart](./docs/quickstart.md)
-- [FAQ](./docs/faq.md)
+- [FAQ](./docs/references/faq.md)
 
 ## 📜 ロードマップ
 
@@ -268,4 +275,4 @@ $ bash ./entrypoint.sh
 
 ## 🙌 コントリビュート
 
-RAGFlow はオープンソースのコラボレーションによって発展してきました。この精神に基づき、私たちはコミュニティからの多様なコントリビュートを受け入れています。 参加を希望される方は、まず[コントリビューションガイド](https://github.com/infiniflow/ragflow/blob/main/docs/CONTRIBUTING.md)をご覧ください。
+RAGFlow はオープンソースのコラボレーションによって発展してきました。この精神に基づき、私たちはコミュニティからの多様なコントリビュートを受け入れています。 参加を希望される方は、まず[コントリビューションガイド](./docs/references/CONTRIBUTING.md)をご覧ください。

README_zh.md

@@ -17,10 +17,9 @@
     <a href="https://demo.ragflow.io" target="_blank">
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"></a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.6.0-brightgreen"
-            alt="docker pull infiniflow/ragflow:v0.6.0"></a>
-      <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
-    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=1570EF" alt="license">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.7.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.7.0"></a>
+    <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
+    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=2e6cc4" alt="license">
   </a>
 </p>
 
@@ -28,17 +27,24 @@
 
 [RAGFlow](https://ragflow.io/) 是一款基于深度文档理解构建的开源 RAG（Retrieval-Augmented Generation）引擎。RAGFlow 可以为各种规模的企业及个人提供一套精简的 RAG 工作流程，结合大语言模型（LLM）针对用户各类不同的复杂格式数据提供可靠的问答以及有理有据的引用。
 
+## 🎮 Demo 试用
+
+请登录网址 [https://demo.ragflow.io](https://demo.ragflow.io) 试用 demo。
+
 ## 📌 近期更新
 
+- 2024-05-30 集成 [BCE](https://github.com/netease-youdao/BCEmbedding), [BGE](https://github.com/FlagOpen/FlagEmbedding) 和 [Colbert](https://github.com/stanford-futuredata/ColBERT) 重排序模型。
+- 2024-05-28 集成大模型 Baichuan 和火山方舟。
+- 2024-05-23 实现 [RAPTOR](https://arxiv.org/html/2401.18059v1) 提供更好的文本检索。
 - 2024-05-21 支持流式结果输出和文本块获取API。
 - 2024-05-15 集成大模型 OpenAI GPT-4o。
 - 2024-05-08 集成大模型 DeepSeek。
 - 2024-04-26 增添了'文件管理'功能。
-- 2024-04-19 支持对话 API ([更多](./docs/conversation_api.md))。
+- 2024-04-19 支持对话 API ([更多](./docs/references/api.md))。
 - 2024-04-16 集成嵌入模型 [BCEmbedding](https://github.com/netease-youdao/BCEmbedding) 和 专为轻型和高速嵌入而设计的 [FastEmbed](https://github.com/qdrant/fastembed)。
-- 2024-04-11 支持用 [Xinference](./docs/xinference.md) 本地化部署大模型。
+- 2024-04-11 支持用 [Xinference](./docs/guides/deploy_local_llm.md) 本地化部署大模型。
 - 2024-04-10 为‘Laws’版面分析增加了底层模型。
-- 2024-04-08 支持用 [Ollama](./docs/ollama.md) 本地化部署大模型。
+- 2024-04-08 支持用 [Ollama](./docs/guides/deploy_local_llm.md) 本地化部署大模型。
 - 2024-04-07 支持中文界面。
 
 ## 🌟 主要功能
@@ -87,7 +93,7 @@
 
 ### 🚀 启动服务器
 
-1. 确保 `vm.max_map_count` 不小于 262144 【[更多](./docs/max_map_count.md)】：
+1. 确保 `vm.max_map_count` 不小于 262144 【[更多](./docs/guides/max_map_count.md)】：
 
    > 如需确认 `vm.max_map_count` 的大小：
    >
@@ -122,7 +128,7 @@
    $ docker compose -f docker-compose-CN.yml up -d
    ```
 
-   > 请注意，运行上述命令会自动下载 RAGFlow 的开发版本 docker 镜像。如果你想下载并运行特定版本的 docker 镜像，请在 docker/.env 文件中找到 RAGFLOW_VERSION 变量，将其改为对应版本。例如 RAGFLOW_VERSION=v0.6.0，然后运行上述命令。
+   > 请注意，运行上述命令会自动下载 RAGFlow 的开发版本 docker 镜像。如果你想下载并运行特定版本的 docker 镜像，请在 docker/.env 文件中找到 RAGFLOW_VERSION 变量，将其改为对应版本。例如 RAGFLOW_VERSION=v0.7.0，然后运行上述命令。
 
    > 核心镜像文件大约 9 GB，可能需要一定时间拉取。请耐心等待。
 
@@ -153,7 +159,7 @@
    > 上面这个例子中，您只需输入 http://IP_OF_YOUR_MACHINE 即可：未改动过配置则无需输入端口（默认的 HTTP 服务端口 80）。
 6. 在 [service_conf.yaml](./docker/service_conf.yaml) 文件的 `user_default_llm` 栏配置 LLM factory，并在 `API_KEY` 栏填写和你选择的大模型相对应的 API key。
 
-   > 详见 [./docs/llm_api_key_setup.md](./docs/llm_api_key_setup.md)。
+   > 详见 [./docs/guides/llm_api_key_setup.md](./docs/guides/llm_api_key_setup.md)。
 
    _好戏开始，接着奏乐接着舞！_
 
@@ -184,7 +190,7 @@
 ```bash
 $ git clone https://github.com/infiniflow/ragflow.git
 $ cd ragflow/
-$ docker build -t infiniflow/ragflow:v0.6.0 .
+$ docker build -t infiniflow/ragflow:v0.7.0 .
 $ cd ragflow/docker
 $ chmod +x ./entrypoint.sh
 $ docker compose up -d
@@ -274,7 +280,7 @@ $ systemctl start nginx
 ## 📚 技术文档
 
 - [Quickstart](./docs/quickstart.md)
-- [FAQ](./docs/faq.md)
+- [FAQ](./docs/references/faq.md)
 
 ## 📜 路线图
 
@@ -287,7 +293,7 @@ $ systemctl start nginx
 
 ## 🙌 贡献指南
 
-RAGFlow 只有通过开源协作才能蓬勃发展。秉持这一精神,我们欢迎来自社区的各种贡献。如果您有意参与其中,请查阅我们的[贡献者指南](https://github.com/infiniflow/ragflow/blob/main/docs/CONTRIBUTING.md) 。
+RAGFlow 只有通过开源协作才能蓬勃发展。秉持这一精神,我们欢迎来自社区的各种贡献。如果您有意参与其中,请查阅我们的[贡献者指南](./docs/references/CONTRIBUTING.md) 。
 
 ## 👥 加入社区
 

api/apps/api_app.py

@@ -20,8 +20,8 @@ from datetime import datetime, timedelta
 from flask import request, Response
 from flask_login import login_required, current_user
 
-from api.db import FileType, ParserType
-from api.db.db_models import APIToken, API4Conversation, Task
+from api.db import FileType, ParserType, FileSource
+from api.db.db_models import APIToken, API4Conversation, Task, File
 from api.db.services import duplicate_name
 from api.db.services.api_service import APITokenService, API4ConversationService
 from api.db.services.dialog_service import DialogService, chat
@@ -31,7 +31,7 @@ from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import queue_tasks, TaskService
 from api.db.services.user_service import UserTenantService
-from api.settings import RetCode
+from api.settings import RetCode, retrievaler
 from api.utils import get_uuid, current_timestamp, datetime_format
 from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request
 from itsdangerous import URLSafeTimedSerializer
@@ -39,9 +39,6 @@ from itsdangerous import URLSafeTimedSerializer
 from api.utils.file_utils import filename_type, thumbnail
 from rag.utils.minio_conn import MINIO
 
-from rag.utils.es_conn import ELASTICSEARCH
-from rag.nlp import search
-from elasticsearch_dsl import Q
 
 def generate_confirmation_token(tenent_id):
     serializer = URLSafeTimedSerializer(tenent_id)
@@ -206,6 +203,9 @@ def completion():
             try:
                 for ans in chat(dia, msg, True, **req):
                     fillin_conv(ans)
+                    for chunk_i in ans['reference'].get('chunks', []):
+                        chunk_i['doc_name'] = chunk_i['docnm_kwd']
+                        chunk_i.pop('docnm_kwd')
                     yield "data:"+json.dumps({"retcode": 0, "retmsg": "", "data": ans}, ensure_ascii=False) + "\n\n"
                 API4ConversationService.append_message(conv.id, conv.to_dict())
             except Exception as e:
@@ -228,6 +228,11 @@ def completion():
                 fillin_conv(ans)
                 API4ConversationService.append_message(conv.id, conv.to_dict())
                 break
+
+            for chunk_i in answer['reference'].get('chunks',[]):
+                chunk_i['doc_name'] = chunk_i['docnm_kwd']
+                chunk_i.pop('docnm_kwd')
+
             return get_json_result(data=answer)
 
     except Exception as e:
@@ -242,7 +247,13 @@ def get(conversation_id):
         if not e:
             return get_data_error_result(retmsg="Conversation not found!")
 
-        return get_json_result(data=conv.to_dict())
+        conv = conv.to_dict()
+        for referenct_i in conv['reference']:
+            for chunk_i in referenct_i['chunks']:
+                if 'docnm_kwd' in chunk_i.keys():
+                    chunk_i['doc_name'] = chunk_i['docnm_kwd']
+                    chunk_i.pop('docnm_kwd')
+        return get_json_result(data=conv)
     except Exception as e:
         return server_error_response(e)
 
@@ -369,27 +380,199 @@ def list_chunks():
     try:
         if "doc_name" in form_data.keys():
             tenant_id = DocumentService.get_tenant_id_by_name(form_data['doc_name'])
-            q = Q("match", docnm_kwd=form_data['doc_name'])
+            doc_id = DocumentService.get_doc_id_by_doc_name(form_data['doc_name'])
 
         elif "doc_id" in form_data.keys():
             tenant_id = DocumentService.get_tenant_id(form_data['doc_id'])
-            q = Q("match", doc_id=form_data['doc_id'])
+            doc_id = form_data['doc_id']
         else:
             return get_json_result(
                 data=False,retmsg="Can't find doc_name or doc_id"
             )
 
-        res_es_search = ELASTICSEARCH.search(q,idxnm=search.index_name(tenant_id),timeout="600s")
-
-        res = [{} for _ in range(len(res_es_search['hits']['hits']))]
-
-        for index , chunk in enumerate(res_es_search['hits']['hits']):
-            res[index]['doc_name'] = chunk['_source']['docnm_kwd']
-            res[index]['content'] = chunk['_source']['content_with_weight']
-            if 'img_id' in chunk['_source'].keys():
-                res[index]['img_id'] = chunk['_source']['img_id']
+        res = retrievaler.chunk_list(doc_id=doc_id, tenant_id=tenant_id)
+        res = [
+            {
+                "content": res_item["content_with_weight"],
+                "doc_name": res_item["docnm_kwd"],
+                "img_id": res_item["img_id"]
+            } for res_item in res
+        ]
 
     except Exception as e:
         return server_error_response(e)
 
     return get_json_result(data=res)
+
+
+@manager.route('/list_kb_docs', methods=['POST'])
+# @login_required
+def list_kb_docs():
+    token = request.headers.get('Authorization').split()[1]
+    objs = APIToken.query(token=token)
+    if not objs:
+        return get_json_result(
+            data=False, retmsg='Token is not valid!"', retcode=RetCode.AUTHENTICATION_ERROR)
+
+    tenant_id = objs[0].tenant_id
+    kb_name = request.form.get("kb_name").strip()
+
+    try:
+        e, kb = KnowledgebaseService.get_by_name(kb_name, tenant_id)
+        if not e:
+            return get_data_error_result(
+                retmsg="Can't find this knowledgebase!")
+        kb_id = kb.id
+
+    except Exception as e:
+        return server_error_response(e)
+
+    page_number = int(request.form.get("page", 1))
+    items_per_page = int(request.form.get("page_size", 15))
+    orderby = request.form.get("orderby", "create_time")
+    desc = request.form.get("desc", True)
+    keywords = request.form.get("keywords", "")
+
+    try:
+        docs, tol = DocumentService.get_by_kb_id(
+            kb_id, page_number, items_per_page, orderby, desc, keywords)
+        docs = [{"doc_id": doc['id'], "doc_name": doc['name']} for doc in docs]
+
+        return get_json_result(data={"total": tol, "docs": docs})
+    
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route('/document', methods=['DELETE'])
+# @login_required
+def document_rm():
+    token = request.headers.get('Authorization').split()[1]
+    objs = APIToken.query(token=token)
+    if not objs:
+        return get_json_result(
+            data=False, retmsg='Token is not valid!"', retcode=RetCode.AUTHENTICATION_ERROR)
+
+    tenant_id = objs[0].tenant_id
+    req = request.json
+    doc_ids = []
+    try:
+        doc_ids = [DocumentService.get_doc_id_by_doc_name(doc_name) for doc_name in req.get("doc_names", [])]
+        for doc_id in req.get("doc_ids", []):
+            if doc_id not in doc_ids:
+                doc_ids.append(doc_id)
+
+        if not doc_ids:
+            return get_json_result(
+                data=False, retmsg="Can't find doc_names or doc_ids"
+            )
+
+    except Exception as e:
+        return server_error_response(e)
+
+    root_folder = FileService.get_root_folder(tenant_id)
+    pf_id = root_folder["id"]
+    FileService.init_knowledgebase_docs(pf_id, tenant_id)
+
+    errors = ""
+    for doc_id in doc_ids:
+        try:
+            e, doc = DocumentService.get_by_id(doc_id)
+            if not e:
+                return get_data_error_result(retmsg="Document not found!")
+            tenant_id = DocumentService.get_tenant_id(doc_id)
+            if not tenant_id:
+                return get_data_error_result(retmsg="Tenant not found!")
+
+            b, n = File2DocumentService.get_minio_address(doc_id=doc_id)
+
+            if not DocumentService.remove_document(doc, tenant_id):
+                return get_data_error_result(
+                    retmsg="Database error (Document removal)!")
+
+            f2d = File2DocumentService.get_by_document_id(doc_id)
+            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
+            File2DocumentService.delete_by_document_id(doc_id)
+
+            MINIO.rm(b, n)
+        except Exception as e:
+            errors += str(e)
+
+    if errors:
+        return get_json_result(data=False, retmsg=errors, retcode=RetCode.SERVER_ERROR)
+
+    return get_json_result(data=True)
+
+
+@manager.route('/completion_aibotk', methods=['POST'])
+@validate_request("Authorization", "conversation_id", "word")
+def completion_faq():
+    import base64
+    req = request.json
+
+    token = req["Authorization"]
+    objs = APIToken.query(token=token)
+    if not objs:
+        return get_json_result(
+            data=False, retmsg='Token is not valid!"', retcode=RetCode.AUTHENTICATION_ERROR)
+
+    e, conv = API4ConversationService.get_by_id(req["conversation_id"])
+    if not e:
+        return get_data_error_result(retmsg="Conversation not found!")
+    if "quote" not in req: req["quote"] = True
+
+    msg = []
+    msg.append({"role": "user", "content": req["word"]})
+
+    try:
+        conv.message.append(msg[-1])
+        e, dia = DialogService.get_by_id(conv.dialog_id)
+        if not e:
+            return get_data_error_result(retmsg="Dialog not found!")
+        del req["conversation_id"]
+
+        if not conv.reference:
+            conv.reference = []
+        conv.message.append({"role": "assistant", "content": ""})
+        conv.reference.append({"chunks": [], "doc_aggs": []})
+
+        def fillin_conv(ans):
+            nonlocal conv
+            if not conv.reference:
+                conv.reference.append(ans["reference"])
+            else: conv.reference[-1] = ans["reference"]
+            conv.message[-1] = {"role": "assistant", "content": ans["answer"]}
+
+        data_type_picture = {
+            "type": 3,
+            "url": "base64 content"
+        }
+        data = [
+            {
+                "type": 1,
+                "content": ""
+            }
+        ]
+        for ans in chat(dia, msg, stream=False, **req):
+            # answer = ans
+            data[0]["content"] += re.sub(r'##\d\$\$', '', ans["answer"])
+            fillin_conv(ans)
+            API4ConversationService.append_message(conv.id, conv.to_dict())
+
+            chunk_idxs = [int(match[2]) for match in re.findall(r'##\d\$\$', ans["answer"])]
+            for chunk_idx in chunk_idxs[:1]:
+                if ans["reference"]["chunks"][chunk_idx]["img_id"]:
+                    try:
+                        bkt, nm = ans["reference"]["chunks"][chunk_idx]["img_id"].split("-")
+                        response = MINIO.get(bkt, nm)
+                        data_type_picture["url"] = base64.b64encode(response).decode('utf-8')
+                        data.append(data_type_picture)
+                    except Exception as e:
+                        return server_error_response(e)
+            break
+
+        response = {"code": 200, "msg": "success", "data": data}
+        return response
+
+    except Exception as e:
+        return server_error_response(e)

api/apps/chunk_app.py

@@ -150,7 +150,7 @@ def set():
             if len(arr) != 2:
                 return get_data_error_result(
                     retmsg="Q&A must be separated by TAB/ENTER key.")
-            q, a = rmPrefix(arr[0]), rmPrefix[arr[1]]
+            q, a = rmPrefix(arr[0]), rmPrefix(arr[1])
             d = beAdoc(d, arr[0], arr[1], not any(
                 [rag_tokenizer.is_chinese(t) for t in q + a]))
 
@@ -229,6 +229,9 @@ def create():
         v = 0.1 * v[0] + 0.9 * v[1]
         d["q_%d_vec" % len(v)] = v.tolist()
         ELASTICSEARCH.upsert([d], search.index_name(tenant_id))
+
+        DocumentService.increment_chunk_num(
+            doc.id, doc.kb_id, c, 1, 0)
         return get_json_result(data={"chunk_id": chunck_id})
     except Exception as e:
         return server_error_response(e)
@@ -254,8 +257,15 @@ def retrieval_test():
 
         embd_mdl = TenantLLMService.model_instance(
             kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
-        ranks = retrievaler.retrieval(question, embd_mdl, kb.tenant_id, [kb_id], page, size, similarity_threshold,
-                                      vector_similarity_weight, top, doc_ids)
+
+        rerank_mdl = None
+        if req.get("rerank_id"):
+            rerank_mdl = TenantLLMService.model_instance(
+                kb.tenant_id, LLMType.RERANK.value, llm_name=req["rerank_id"])
+
+        ranks = retrievaler.retrieval(question, embd_mdl, kb.tenant_id, [kb_id], page, size,
+                                      similarity_threshold, vector_similarity_weight, top,
+                                      doc_ids, rerank_mdl=rerank_mdl)
         for c in ranks["chunks"]:
             if "vector" in c:
                 del c["vector"]

api/apps/dialog_app.py

@@ -33,8 +33,12 @@ def set_dialog():
     name = req.get("name", "New Dialog")
     description = req.get("description", "A helpful Dialog")
     top_n = req.get("top_n", 6)
+    top_k = req.get("top_k", 1024)
+    rerank_id = req.get("rerank_id", "")
+    if not rerank_id: req["rerank_id"] = ""
     similarity_threshold = req.get("similarity_threshold", 0.1)
     vector_similarity_weight = req.get("vector_similarity_weight", 0.3)
+    if vector_similarity_weight is None: vector_similarity_weight = 0.3
     llm_setting = req.get("llm_setting", {})
     default_prompt = {
         "system": """你是一个智能助手，请总结知识库的内容来回答问题，请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时，你的回答必须包括“知识库中未找到您要的答案！”这句话。回答需要考虑聊天历史。
@@ -83,6 +87,8 @@ def set_dialog():
                 "llm_setting": llm_setting,
                 "prompt_config": prompt_config,
                 "top_n": top_n,
+                "top_k": top_k,
+                "rerank_id": rerank_id,
                 "similarity_threshold": similarity_threshold,
                 "vector_similarity_weight": vector_similarity_weight
             }

api/apps/llm_app.py

@@ -20,7 +20,7 @@ from api.utils.api_utils import server_error_response, get_data_error_result, va
 from api.db import StatusEnum, LLMType
 from api.db.db_models import TenantLLM
 from api.utils.api_utils import get_json_result
-from rag.llm import EmbeddingModel, ChatModel
+from rag.llm import EmbeddingModel, ChatModel, RerankModel
 
 
 @manager.route('/factories', methods=['GET'])
@@ -28,7 +28,7 @@ from rag.llm import EmbeddingModel, ChatModel
 def factories():
     try:
         fac = LLMFactoriesService.get_all()
-        return get_json_result(data=[f.to_dict() for f in fac if f.name not in ["Youdao", "FastEmbed"]])
+        return get_json_result(data=[f.to_dict() for f in fac if f.name not in ["Youdao", "FastEmbed", "BAAI"]])
     except Exception as e:
         return server_error_response(e)
 
@@ -39,17 +39,18 @@ def factories():
 def set_api_key():
     req = request.json
     # test if api key works
-    chat_passed = False
+    chat_passed, embd_passed, rerank_passed = False, False, False
     factory = req["llm_factory"]
     msg = ""
     for llm in LLMService.query(fid=factory):
-        if llm.model_type == LLMType.EMBEDDING.value:
+        if not embd_passed and llm.model_type == LLMType.EMBEDDING.value:
             mdl = EmbeddingModel[factory](
                 req["api_key"], llm.llm_name, base_url=req.get("base_url"))
             try:
                 arr, tc = mdl.encode(["Test if the api key is available"])
                 if len(arr[0]) == 0 or tc == 0:
                     raise Exception("Fail")
+                embd_passed = True
             except Exception as e:
                 msg += f"\nFail to access embedding model({llm.llm_name}) using this api key." + str(e)
         elif not chat_passed and llm.model_type == LLMType.CHAT.value:
@@ -60,10 +61,21 @@ def set_api_key():
                                  "temperature": 0.9})
                 if not tc:
                     raise Exception(m)
-                chat_passed = True
             except Exception as e:
                 msg += f"\nFail to access model({llm.llm_name}) using this api key." + str(
                     e)
+            chat_passed = True
+        elif not rerank_passed and llm.model_type == LLMType.RERANK:
+            mdl = RerankModel[factory](
+                req["api_key"], llm.llm_name, base_url=req.get("base_url"))
+            try:
+                arr, tc = mdl.similarity("What's the weather?", ["Is it sunny today?"])
+                if len(arr) == 0 or tc == 0:
+                    raise Exception("Fail")
+            except Exception as e:
+                msg += f"\nFail to access model({llm.llm_name}) using this api key." + str(
+                    e)
+            rerank_passed = True
 
     if msg:
         return get_data_error_result(retmsg=msg)
@@ -96,16 +108,29 @@ def set_api_key():
 @validate_request("llm_factory", "llm_name", "model_type")
 def add_llm():
     req = request.json
+    factory = req["llm_factory"]
+    # For VolcEngine, due to its special authentication method
+    # Assemble volc_ak, volc_sk, endpoint_id into api_key
+    if factory == "VolcEngine":
+        temp = list(eval(req["llm_name"]).items())[0]
+        llm_name = temp[0]
+        endpoint_id = temp[1]
+        api_key = '{' + f'"volc_ak": "{req.get("volc_ak", "")}", ' \
+                        f'"volc_sk": "{req.get("volc_sk", "")}", ' \
+                        f'"ep_id": "{endpoint_id}", ' + '}'
+    else:
+        llm_name = req["llm_name"]
+        api_key = "xxxxxxxxxxxxxxx"
+
     llm = {
         "tenant_id": current_user.id,
-        "llm_factory": req["llm_factory"],
+        "llm_factory": factory,
         "model_type": req["model_type"],
-        "llm_name": req["llm_name"],
+        "llm_name": llm_name,
         "api_base": req.get("api_base", ""),
-        "api_key": "xxxxxxxxxxxxxxx"
+        "api_key": api_key
     }
 
-    factory = req["llm_factory"]
     msg = ""
     if llm["model_type"] == LLMType.EMBEDDING.value:
         mdl = EmbeddingModel[factory](
@@ -118,7 +143,10 @@ def add_llm():
             msg += f"\nFail to access embedding model({llm['llm_name']})." + str(e)
     elif llm["model_type"] == LLMType.CHAT.value:
         mdl = ChatModel[factory](
-            key=None, model_name=llm["llm_name"], base_url=llm["api_base"])
+            key=llm['api_key'] if factory == "VolcEngine" else None,
+            model_name=llm["llm_name"],
+            base_url=llm["api_base"]
+        )
         try:
             m, tc = mdl.chat(None, [{"role": "user", "content": "Hello! How are you doing!"}], {
                              "temperature": 0.9})
@@ -134,7 +162,6 @@ def add_llm():
     if msg:
         return get_data_error_result(retmsg=msg)
 
-
     if not TenantLLMService.filter_update(
             [TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == factory, TenantLLM.llm_name == llm["llm_name"]], llm):
         TenantLLMService.save(**llm)
@@ -184,7 +211,7 @@ def list_app():
         llms = [m.to_dict()
                 for m in llms if m.status == StatusEnum.VALID.value]
         for m in llms:
-            m["available"] = m["fid"] in facts or m["llm_name"].lower() == "flag-embedding" or m["fid"] in ["Youdao","FastEmbed"]
+            m["available"] = m["fid"] in facts or m["llm_name"].lower() == "flag-embedding" or m["fid"] in ["Youdao","FastEmbed", "BAAI"]
 
         llm_set = set([m["llm_name"] for m in llms])
         for o in objs:

api/apps/system_app.py

@@ -60,7 +60,8 @@ def status():
     st = timer()
     try:
         qinfo = REDIS_CONN.health(SVR_QUEUE_NAME)
-        res["redis"] = {"status": "green", "elapsed": "{:.1f}".format((timer() - st)*1000.), "pending": qinfo["pending"]}
+        res["redis"] = {"status": "green", "elapsed": "{:.1f}".format((timer() - st)*1000.),
+                        "pending": qinfo.get("pending", 0)}
     except Exception as e:
         res["redis"] = {"status": "red", "elapsed": "{:.1f}".format((timer() - st)*1000.), "error": str(e)}
 

api/apps/user_app.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import json
 import re
 from datetime import datetime
 
@@ -25,8 +26,9 @@ from api.db.services.llm_service import TenantLLMService, LLMService
 from api.utils.api_utils import server_error_response, validate_request
 from api.utils import get_uuid, get_format_time, decrypt, download_img, current_timestamp, datetime_format
 from api.db import UserTenantRole, LLMType, FileType
-from api.settings import RetCode, GITHUB_OAUTH, CHAT_MDL, EMBEDDING_MDL, ASR_MDL, IMAGE2TEXT_MDL, PARSERS, API_KEY, \
-    LLM_FACTORY, LLM_BASE_URL
+from api.settings import RetCode, GITHUB_OAUTH, FEISHU_OAUTH, CHAT_MDL, EMBEDDING_MDL, ASR_MDL, IMAGE2TEXT_MDL, PARSERS, \
+    API_KEY, \
+    LLM_FACTORY, LLM_BASE_URL, RERANK_MDL
 from api.db.services.user_service import UserService, TenantService, UserTenantService
 from api.db.services.file_service import FileService
 from api.settings import stat_logger
@@ -287,7 +289,8 @@ def user_register(user_id, user):
         "embd_id": EMBEDDING_MDL,
         "asr_id": ASR_MDL,
         "parser_ids": PARSERS,
-        "img2txt_id": IMAGE2TEXT_MDL
+        "img2txt_id": IMAGE2TEXT_MDL,
+        "rerank_id": RERANK_MDL
     }
     usr_tenant = {
         "tenant_id": user_id,

api/db/__init__.py

@@ -54,6 +54,7 @@ class LLMType(StrEnum):
     EMBEDDING = 'embedding'
     SPEECH2TEXT = 'speech2text'
     IMAGE2TEXT = 'image2text'
+    RERANK = 'rerank'
 
 
 class ChatStyle(StrEnum):

api/db/db_models.py

@@ -386,7 +386,7 @@ class User(DataBaseModel, UserMixin):
         max_length=32,
         null=True,
         help_text="English|Chinese",
-        default="Chinese" if "zh_CN" in os.getenv("LANG", "") else "English")     
+        default="Chinese" if "zh_CN" in os.getenv("LANG", "") else "English")
     color_schema = CharField(
         max_length=32,
         null=True,
@@ -437,6 +437,10 @@ class Tenant(DataBaseModel):
         max_length=128,
         null=False,
         help_text="default image to text model ID")
+    rerank_id = CharField(
+        max_length=128,
+        null=False,
+        help_text="default rerank model ID")
     parser_ids = CharField(
         max_length=256,
         null=False,
@@ -771,11 +775,16 @@ class Dialog(DataBaseModel):
     similarity_threshold = FloatField(default=0.2)
     vector_similarity_weight = FloatField(default=0.3)
     top_n = IntegerField(default=6)
+    top_k = IntegerField(default=1024)
     do_refer = CharField(
         max_length=1,
         null=False,
         help_text="it needs to insert reference index into answer or not",
         default="1")
+    rerank_id = CharField(
+        max_length=128,
+        null=False,
+        help_text="default rerank model ID")
 
     kb_ids = JSONField(null=False, default=[])
     status = CharField(
@@ -825,11 +834,29 @@ class API4Conversation(DataBaseModel):
 
 
 def migrate_db():
-    try:
         with DB.transaction():
             migrator = MySQLMigrator(DB)
-            migrate(
-                migrator.add_column('file', 'source_type', CharField(max_length=128, null=False, default="", help_text="where dose this document come from"))
-            )
-    except Exception as e:
-        pass
+            try:
+                migrate(
+                    migrator.add_column('file', 'source_type', CharField(max_length=128, null=False, default="", help_text="where dose this document come from"))
+                )
+            except Exception as e:
+                pass
+            try:
+                migrate(
+                    migrator.add_column('tenant', 'rerank_id', CharField(max_length=128, null=False, default="BAAI/bge-reranker-v2-m3", help_text="default rerank model ID"))
+                )
+            except Exception as e:
+                pass
+            try:
+                migrate(
+                    migrator.add_column('dialog', 'rerank_id', CharField(max_length=128, null=False, default="", help_text="default rerank model ID"))
+                )
+            except Exception as e:
+                pass
+            try:
+                migrate(
+                    migrator.add_column('dialog', 'top_k', IntegerField(default=1024))
+                )
+            except Exception as e:
+                pass

api/db/init_data.py

@@ -132,7 +132,27 @@ factory_infos = [{
     "logo": "",
     "tags": "LLM",
     "status": "1",
-},
+},{
+    "name": "VolcEngine",
+    "logo": "",
+    "tags": "LLM, TEXT EMBEDDING",
+    "status": "1",
+},{
+    "name": "BaiChuan",
+    "logo": "",
+    "tags": "LLM,TEXT EMBEDDING",
+    "status": "1",
+},{
+    "name": "Jina",
+    "logo": "",
+    "tags": "TEXT EMBEDDING, TEXT RE-RANK",
+    "status": "1",
+},{
+    "name": "BAAI",
+    "logo": "",
+    "tags": "TEXT EMBEDDING, TEXT RE-RANK",
+    "status": "1",
+}
     # {
     #     "name": "文心一言",
     #     "logo": "",
@@ -357,6 +377,13 @@ def init_llm_factory():
             "max_tokens": 512,
             "model_type": LLMType.EMBEDDING.value
         },
+        {
+            "fid": factory_infos[7]["name"],
+            "llm_name": "maidalun1020/bce-reranker-base_v1",
+            "tags": "RE-RANK, 8K",
+            "max_tokens": 8196,
+            "model_type": LLMType.RERANK.value
+        },
         # ------------------------ DeepSeek -----------------------
         {
             "fid": factory_infos[8]["name"],
@@ -372,6 +399,143 @@ def init_llm_factory():
             "max_tokens": 16385,
             "model_type": LLMType.CHAT.value
         },
+        # ------------------------ VolcEngine -----------------------
+        {
+            "fid": factory_infos[9]["name"],
+            "llm_name": "Skylark2-pro-32k",
+            "tags": "LLM,CHAT,32k",
+            "max_tokens": 32768,
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[9]["name"],
+            "llm_name": "Skylark2-pro-4k",
+            "tags": "LLM,CHAT,4k",
+            "max_tokens": 4096,
+            "model_type": LLMType.CHAT.value
+        },
+        # ------------------------ BaiChuan -----------------------
+        {
+            "fid": factory_infos[10]["name"],
+            "llm_name": "Baichuan2-Turbo",
+            "tags": "LLM,CHAT,32K",
+            "max_tokens": 32768,
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[10]["name"],
+            "llm_name": "Baichuan2-Turbo-192k",
+            "tags": "LLM,CHAT,192K",
+            "max_tokens": 196608,
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[10]["name"],
+            "llm_name": "Baichuan3-Turbo",
+            "tags": "LLM,CHAT,32K",
+            "max_tokens": 32768,
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[10]["name"],
+            "llm_name": "Baichuan3-Turbo-128k",
+            "tags": "LLM,CHAT,128K",
+            "max_tokens": 131072,
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[10]["name"],
+            "llm_name": "Baichuan4",
+            "tags": "LLM,CHAT,128K",
+            "max_tokens": 131072,
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[10]["name"],
+            "llm_name": "Baichuan-Text-Embedding",
+            "tags": "TEXT EMBEDDING",
+            "max_tokens": 512,
+            "model_type": LLMType.EMBEDDING.value
+        },
+        # ------------------------ Jina -----------------------
+        {
+            "fid": factory_infos[11]["name"],
+            "llm_name": "jina-reranker-v1-base-en",
+            "tags": "RE-RANK,8k",
+            "max_tokens": 8196,
+            "model_type": LLMType.RERANK.value
+        },
+        {
+            "fid": factory_infos[11]["name"],
+            "llm_name": "jina-reranker-v1-turbo-en",
+            "tags": "RE-RANK,8k",
+            "max_tokens": 8196,
+            "model_type": LLMType.RERANK.value
+        },
+        {
+            "fid": factory_infos[11]["name"],
+            "llm_name": "jina-reranker-v1-tiny-en",
+            "tags": "RE-RANK,8k",
+            "max_tokens": 8196,
+            "model_type": LLMType.RERANK.value
+        },
+        {
+            "fid": factory_infos[11]["name"],
+            "llm_name": "jina-colbert-v1-en",
+            "tags": "RE-RANK,8k",
+            "max_tokens": 8196,
+            "model_type": LLMType.RERANK.value
+        },
+        {
+            "fid": factory_infos[11]["name"],
+            "llm_name": "jina-embeddings-v2-base-en",
+            "tags": "TEXT EMBEDDING",
+            "max_tokens": 8196,
+            "model_type": LLMType.EMBEDDING.value
+        },
+        {
+            "fid": factory_infos[11]["name"],
+            "llm_name": "jina-embeddings-v2-base-de",
+            "tags": "TEXT EMBEDDING",
+            "max_tokens": 8196,
+            "model_type": LLMType.EMBEDDING.value
+        },
+        {
+            "fid": factory_infos[11]["name"],
+            "llm_name": "jina-embeddings-v2-base-es",
+            "tags": "TEXT EMBEDDING",
+            "max_tokens": 8196,
+            "model_type": LLMType.EMBEDDING.value
+        },
+        {
+            "fid": factory_infos[11]["name"],
+            "llm_name": "jina-embeddings-v2-base-code",
+            "tags": "TEXT EMBEDDING",
+            "max_tokens": 8196,
+            "model_type": LLMType.EMBEDDING.value
+        },
+        {
+            "fid": factory_infos[11]["name"],
+            "llm_name": "jina-embeddings-v2-base-zh",
+            "tags": "TEXT EMBEDDING",
+            "max_tokens": 8196,
+            "model_type": LLMType.EMBEDDING.value
+        },
+        # ------------------------ BAAI -----------------------
+        {
+            "fid": factory_infos[12]["name"],
+            "llm_name": "BAAI/bge-large-zh-v1.5",
+            "tags": "TEXT EMBEDDING,",
+            "max_tokens": 1024,
+            "model_type": LLMType.EMBEDDING.value
+        },
+        {
+            "fid": factory_infos[12]["name"],
+            "llm_name": "BAAI/bge-reranker-v2-m3",
+            "tags": "RE-RANK,2k",
+            "max_tokens": 2048,
+            "model_type": LLMType.RERANK.value
+        },
     ]
     for info in factory_infos:
         try:

api/db/services/dialog_service.py

@@ -58,17 +58,17 @@ def message_fit_in(msg, max_length=4000):
     if c < max_length:
         return c, msg
 
-    ll = num_tokens_from_string(msg_[0].content)
-    l = num_tokens_from_string(msg_[-1].content)
+    ll = num_tokens_from_string(msg_[0]["content"])
+    l = num_tokens_from_string(msg_[-1]["content"])
     if ll / (ll + l) > 0.8:
-        m = msg_[0].content
+        m = msg_[0]["content"]
         m = encoder.decode(encoder.encode(m)[:max_length - l])
-        msg[0].content = m
+        msg[0]["content"] = m
         return max_length, msg
 
-    m = msg_[1].content
+    m = msg_[1]["content"]
     m = encoder.decode(encoder.encode(m)[:max_length - l])
-    msg[1].content = m
+    msg[1]["content"] = m
     return max_length, msg
 
 
@@ -115,11 +115,14 @@ def chat(dialog, messages, stream=True, **kwargs):
     if "knowledge" not in [p["key"] for p in prompt_config["parameters"]]:
         kbinfos = {"total": 0, "chunks": [], "doc_aggs": []}
     else:
+        rerank_mdl = None
+        if dialog.rerank_id:
+            rerank_mdl = LLMBundle(dialog.tenant_id, LLMType.RERANK, dialog.rerank_id)
         kbinfos = retrievaler.retrieval(" ".join(questions), embd_mdl, dialog.tenant_id, dialog.kb_ids, 1, dialog.top_n,
                                         dialog.similarity_threshold,
                                         dialog.vector_similarity_weight,
                                         doc_ids=kwargs["doc_ids"].split(",") if "doc_ids" in kwargs else None,
-                                        top=1024, aggs=False)
+                                        top=1024, aggs=False, rerank_mdl=rerank_mdl)
     knowledges = [ck["content_with_weight"] for ck in kbinfos["chunks"]]
     chat_logger.info(
         "{}->{}".format(" ".join(questions), "\n->".join(knowledges)))
@@ -130,9 +133,13 @@ def chat(dialog, messages, stream=True, **kwargs):
 
     kwargs["knowledge"] = "\n".join(knowledges)
     gen_conf = dialog.llm_setting
-    msg = [{"role": m["role"], "content": m["content"]}
-           for m in messages if m["role"] != "system"]
+
+    msg = [{"role": "system", "content": prompt_config["system"].format(**kwargs)}]
+    msg.extend([{"role": m["role"], "content": m["content"]}
+           for m in messages if m["role"] != "system"])
     used_token_count, msg = message_fit_in(msg, int(max_tokens * 0.97))
+    assert len(msg) >= 2, f"message_fit_in has bug: {msg}"
+
     if "max_tokens" in gen_conf:
         gen_conf["max_tokens"] = min(
             gen_conf["max_tokens"],
@@ -165,14 +172,13 @@ def chat(dialog, messages, stream=True, **kwargs):
 
     if stream:
         answer = ""
-        for ans in chat_mdl.chat_streamly(prompt_config["system"].format(**kwargs), msg, gen_conf):
+        for ans in chat_mdl.chat_streamly(msg[0]["content"], msg[1:], gen_conf):
             answer = ans
             yield {"answer": answer, "reference": {}}
         yield decorate_answer(answer)
     else:
         answer = chat_mdl.chat(
-            prompt_config["system"].format(
-                **kwargs), msg, gen_conf)
+            msg[0]["content"], msg[1:], gen_conf)
         chat_logger.info("User: {}|Assistant: {}".format(
             msg[-1]["content"], answer))
         yield decorate_answer(answer)

api/db/services/document_service.py

@@ -18,8 +18,10 @@ from datetime import datetime
 from elasticsearch_dsl import Q
 from peewee import fn
 
+from api.db.db_utils import bulk_insert_into_db
 from api.settings import stat_logger
-from api.utils import current_timestamp, get_format_time
+from api.utils import current_timestamp, get_format_time, get_uuid
+from rag.settings import SVR_QUEUE_NAME
 from rag.utils.es_conn import ELASTICSEARCH
 from rag.utils.minio_conn import MINIO
 from rag.nlp import search
@@ -30,6 +32,7 @@ from api.db.db_models import Document
 from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db import StatusEnum
+from rag.utils.redis_conn import REDIS_CONN
 
 
 class DocumentService(CommonService):
@@ -110,7 +113,7 @@ class DocumentService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_unfinished_docs(cls):
-        fields = [cls.model.id, cls.model.process_begin_at]
+        fields = [cls.model.id, cls.model.process_begin_at, cls.model.parser_config, cls.model.progress_msg]
         docs = cls.model.select(*fields) \
             .where(
                 cls.model.status == StatusEnum.VALID.value,
@@ -179,6 +182,17 @@ class DocumentService(CommonService):
             return
         return docs[0]["tenant_id"]
 
+    @classmethod
+    @DB.connection_context()
+    def get_doc_id_by_doc_name(cls, doc_name):
+        fields = [cls.model.id]
+        doc_id = cls.model.select(*fields) \
+            .where(cls.model.name == doc_name)
+        doc_id = doc_id.dicts()
+        if not doc_id:
+            return
+        return doc_id[0]["id"]
+
     @classmethod
     @DB.connection_context()
     def get_thumbnails(cls, docids):
@@ -249,7 +263,12 @@ class DocumentService(CommonService):
                     prg = -1
                     status = TaskStatus.FAIL.value
                 elif finished:
-                    status = TaskStatus.DONE.value
+                    if d["parser_config"].get("raptor", {}).get("use_raptor") and d["progress_msg"].lower().find(" raptor")<0:
+                        queue_raptor_tasks(d)
+                        prg *= 0.98
+                        msg.append("------ RAPTOR -------")
+                    else:
+                        status = TaskStatus.DONE.value
 
                 msg = "\n".join(msg)
                 info = {
@@ -271,3 +290,19 @@ class DocumentService(CommonService):
         return len(cls.model.select(cls.model.id).where(
             cls.model.kb_id == kb_id).dicts())
 
+
+def queue_raptor_tasks(doc):
+    def new_task():
+        nonlocal doc
+        return {
+            "id": get_uuid(),
+            "doc_id": doc["id"],
+            "from_page": 0,
+            "to_page": -1,
+            "progress_msg": "Start to do RAPTOR (Recursive Abstractive Processing For Tree-Organized Retrieval)."
+        }
+
+    task = new_task()
+    bulk_insert_into_db(Task, [task], True)
+    task["type"] = "raptor"
+    assert REDIS_CONN.queue_product(SVR_QUEUE_NAME, message=task), "Can't access Redis. Please check the Redis' status."

api/db/services/llm_service.py

@@ -15,7 +15,7 @@
 #
 from api.db.services.user_service import TenantService
 from api.settings import database_logger
-from rag.llm import EmbeddingModel, CvModel, ChatModel
+from rag.llm import EmbeddingModel, CvModel, ChatModel, RerankModel
 from api.db import LLMType
 from api.db.db_models import DB, UserTenant
 from api.db.db_models import LLMFactories, LLM, TenantLLM
@@ -73,21 +73,25 @@ class TenantLLMService(CommonService):
             mdlnm = tenant.img2txt_id
         elif llm_type == LLMType.CHAT.value:
             mdlnm = tenant.llm_id if not llm_name else llm_name
+        elif llm_type == LLMType.RERANK:
+            mdlnm = tenant.rerank_id if not llm_name else llm_name
         else:
             assert False, "LLM type error"
 
         model_config = cls.get_api_key(tenant_id, mdlnm)
         if model_config: model_config = model_config.to_dict()
         if not model_config:
-            if llm_type == LLMType.EMBEDDING.value:
+            if llm_type in [LLMType.EMBEDDING, LLMType.RERANK]:
                 llm = LLMService.query(llm_name=llm_name)
-                if llm and llm[0].fid in ["Youdao", "FastEmbed", "DeepSeek"]:
+                if llm and llm[0].fid in ["Youdao", "FastEmbed", "BAAI"]:
                     model_config = {"llm_factory": llm[0].fid, "api_key":"", "llm_name": llm_name, "api_base": ""}
             if not model_config:
                 if llm_name == "flag-embedding":
                     model_config = {"llm_factory": "Tongyi-Qianwen", "api_key": "",
                                 "llm_name": llm_name, "api_base": ""}
                 else:
+                    if not mdlnm:
+                        raise LookupError(f"Type of {llm_type} model is not set.")
                     raise LookupError("Model({}) not authorized".format(mdlnm))
 
         if llm_type == LLMType.EMBEDDING.value:
@@ -96,6 +100,12 @@ class TenantLLMService(CommonService):
             return EmbeddingModel[model_config["llm_factory"]](
                 model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
 
+        if llm_type == LLMType.RERANK:
+            if model_config["llm_factory"] not in RerankModel:
+                return
+            return RerankModel[model_config["llm_factory"]](
+                model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
+
         if llm_type == LLMType.IMAGE2TEXT.value:
             if model_config["llm_factory"] not in CvModel:
                 return
@@ -125,14 +135,19 @@ class TenantLLMService(CommonService):
             mdlnm = tenant.img2txt_id
         elif llm_type == LLMType.CHAT.value:
             mdlnm = tenant.llm_id if not llm_name else llm_name
+        elif llm_type == LLMType.RERANK:
+            mdlnm = tenant.llm_id if not llm_name else llm_name
         else:
             assert False, "LLM type error"
 
         num = 0
-        for u in cls.query(tenant_id = tenant_id, llm_name=mdlnm):
-            num += cls.model.update(used_tokens = u.used_tokens + used_tokens)\
-                .where(cls.model.tenant_id == tenant_id, cls.model.llm_name == mdlnm)\
-                .execute()
+        try:
+            for u in cls.query(tenant_id = tenant_id, llm_name=mdlnm):
+                num += cls.model.update(used_tokens = u.used_tokens + used_tokens)\
+                    .where(cls.model.tenant_id == tenant_id, cls.model.llm_name == mdlnm)\
+                    .execute()
+        except Exception as e:
+            pass
         return num
 
     @classmethod
@@ -155,6 +170,10 @@ class LLMBundle(object):
             tenant_id, llm_type, llm_name, lang=lang)
         assert self.mdl, "Can't find mole for {}/{}/{}".format(
             tenant_id, llm_type, llm_name)
+        self.max_length = 512
+        for lm in LLMService.query(llm_name=llm_name):
+            self.max_length = lm.max_tokens
+            break
 
     def encode(self, texts: list, batch_size=32):
         emd, used_tokens = self.mdl.encode(texts, batch_size)
@@ -172,6 +191,14 @@ class LLMBundle(object):
                 "Can't update token usage for {}/EMBEDDING".format(self.tenant_id))
         return emd, used_tokens
 
+    def similarity(self, query: str, texts: list):
+        sim, used_tokens = self.mdl.similarity(query, texts)
+        if not TenantLLMService.increase_usage(
+                self.tenant_id, self.llm_type, used_tokens):
+            database_logger.error(
+                "Can't update token usage for {}/RERANK".format(self.tenant_id))
+        return sim, used_tokens
+
     def describe(self, image, max_tokens=300):
         txt, used_tokens = self.mdl.describe(image, max_tokens)
         if not TenantLLMService.increase_usage(

api/db/services/task_service.py

@@ -53,6 +53,7 @@ class TaskService(CommonService):
             Knowledgebase.embd_id,
             Tenant.img2txt_id,
             Tenant.asr_id,
+            Tenant.llm_id,
             cls.model.update_time]
         docs = cls.model.select(*fields) \
             .join(Document, on=(cls.model.doc_id == Document.id)) \
@@ -159,4 +160,4 @@ def queue_tasks(doc, bucket, name):
     DocumentService.begin2parse(doc["id"])
 
     for t in tsks:
-        assert REDIS_CONN.queue_product(SVR_QUEUE_NAME, message=t), "Can't access Redis. Please check the Redis' status."
+        assert REDIS_CONN.queue_product(SVR_QUEUE_NAME, message=t), "Can't access Redis. Please check the Redis' status."

api/db/services/user_service.py

@@ -93,6 +93,7 @@ class TenantService(CommonService):
             cls.model.name,
             cls.model.llm_id,
             cls.model.embd_id,
+            cls.model.rerank_id,
             cls.model.asr_id,
             cls.model.img2txt_id,
             cls.model.parser_ids,

api/settings.py

@@ -89,9 +89,22 @@ default_llm = {
     },
     "DeepSeek": {
         "chat_model": "deepseek-chat",
+        "embedding_model": "",
+        "image2text_model": "",
+        "asr_model": "",
+    },
+    "VolcEngine": {
+        "chat_model": "",
+        "embedding_model": "",
+        "image2text_model": "",
+        "asr_model": "",
+    },
+    "BAAI": {
+        "chat_model": "",
         "embedding_model": "BAAI/bge-large-zh-v1.5",
         "image2text_model": "",
         "asr_model": "",
+        "rerank_model": "BAAI/bge-reranker-v2-m3",
     }
 }
 LLM = get_base_config("user_default_llm", {})
@@ -104,7 +117,8 @@ if LLM_FACTORY not in default_llm:
         f"LLM factory {LLM_FACTORY} has not supported yet, switch to 'Tongyi-Qianwen/QWen' automatically, and please check the API_KEY in service_conf.yaml.")
     LLM_FACTORY = "Tongyi-Qianwen"
 CHAT_MDL = default_llm[LLM_FACTORY]["chat_model"]
-EMBEDDING_MDL = default_llm[LLM_FACTORY]["embedding_model"]
+EMBEDDING_MDL = default_llm["BAAI"]["embedding_model"]
+RERANK_MDL = default_llm["BAAI"]["rerank_model"]
 ASR_MDL = default_llm[LLM_FACTORY]["asr_model"]
 IMAGE2TEXT_MDL = default_llm[LLM_FACTORY]["image2text_model"]
 

api/utils/file_utils.py

@@ -156,7 +156,7 @@ def filename_type(filename):
         return FileType.PDF.value
 
     if re.match(
-            r".*\.(doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt)$", filename):
+             r".*\.(doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|html)$", filename):
         return FileType.DOC.value
 
     if re.match(

deepdoc/parser/__init__.py

@@ -4,3 +4,4 @@ from .pdf_parser import RAGFlowPdfParser as PdfParser, PlainParser
 from .docx_parser import RAGFlowDocxParser as DocxParser
 from .excel_parser import RAGFlowExcelParser as ExcelParser
 from .ppt_parser import RAGFlowPptParser as PptParser
+from .html_parser import RAGFlowHtmlParser as HtmlParser

deepdoc/parser/html_parser.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from rag.nlp import find_codec
+import readability
+import html_text
+import chardet
+
+def get_encoding(file):
+    with open(file,'rb') as f:
+        tmp = chardet.detect(f.read())
+        return tmp['encoding']
+
+class RAGFlowHtmlParser:
+    def __call__(self, fnm, binary=None):
+        txt = ""
+        if binary:
+            encoding = find_codec(binary)
+            txt = binary.decode(encoding, errors="ignore")
+        else:
+            with open(fnm, "r",encoding=get_encoding(fnm)) as f:
+                txt = f.read()
+
+        html_doc = readability.Document(txt)
+        title = html_doc.title()
+        content = html_text.extract_text(html_doc.summary(html_partial=True))
+        txt = f'{title}\n{content}'
+        sections = txt.split("\n")
+        return sections

deepdoc/parser/pdf_parser.py

@@ -392,11 +392,11 @@ class RAGFlowPdfParser:
                 b["text"].strip()[-1] in ",;:'\"，、‘“；：-",
                 len(b["text"].strip()) > 1 and b["text"].strip(
                 )[-2] in ",;:'\"，‘“、；：",
-                b["text"].strip()[0] in "。；？！?”）),，、：",
+                b_["text"].strip() and b_["text"].strip()[0] in "。；？！?”）),，、：",
             ]
             # features for not concating
             feats = [
-                b.get("layoutno", 0) != b.get("layoutno", 0),
+                b.get("layoutno", 0) != b_.get("layoutno", 0),
                 b["text"].strip()[-1] in "。？！?",
                 self.is_english and b["text"].strip()[-1] in ".!?",
                 b["page_number"] == b_["page_number"] and b_["top"] -

deepdoc/vision/postprocess.py

@@ -1,5 +1,5 @@
 import copy
-
+import re
 import numpy as np
 import cv2
 from shapely.geometry import Polygon

docker/.env

@@ -29,7 +29,7 @@ REDIS_PASSWORD=infini_rag_flow
 
 SVR_HTTP_PORT=9380
 
-RAGFLOW_VERSION=0.6.0
+RAGFLOW_VERSION=v0.7.0
 
 TIMEZONE='Asia/Shanghai'
 

docker/README.md

@@ -67,7 +67,7 @@ The serving IP and port inside the docker container. This is not updating until
 Newly signed-up users use LLM configured by this part. Otherwise, user need to configure his own LLM in *setting*.
   
 ### factory
-The LLM suppliers. 'Tongyi-Qianwen', "OpenAI"， "Moonshot" and "ZHIPU-AI" are supported.
+The LLM suppliers. "OpenAI"， "Tongyi-Qianwen", "ZHIPU-AI", "Moonshot", "DeepSeek", "Baichuan", and "VolcEngine" are supported.
 
 ### api_key
 The corresponding API key of your assigned LLM vendor.

docs/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "Get Started",
+  "position": 1,
+  "link": {
+    "type": "generated-index",
+    "description": "RAGFlow Quick Start"
+  }
+}

docs/conversation_api.md

@@ -1,403 +0,0 @@
-# Conversation API Instruction
-
-<div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/df0dcc3d-789a-44f7-89f1-7a5f044ab729" width="830"/>
-</div>
-
-## Base URL
-```buildoutcfg
-https://demo.ragflow.io/v1/
-```
-
-## Authorization
-
-All the APIs are authorized with API-Key. Please keep it safe and private. Don't reveal it in any way from the front-end. 
-The API-Key should put in the header of request:
-```buildoutcfg
-Authorization: Bearer {API_KEY}
-```
-
-## Start a conversation
-
-This should be called whenever there's new user coming to chat.
-### Path: /api/new_conversation
-### Method: GET
-### Parameter:
-
-| name | type | optional | description|
-|------|-------|----|----|
-| user_id| string | No | It's for identifying user in order to search and calculate statistics.|
-
-### Response 
-```json
-{
-    "data": {
-        "create_date": "Fri, 12 Apr 2024 17:26:21 GMT",
-        "create_time": 1712913981857,
-        "dialog_id": "4f0a2e4cb9af11ee9ba20aef05f5e94f",
-        "duration": 0.0,
-        "id": "b9b2e098f8ae11ee9f45fa163e197198",
-        "message": [
-            {
-                "content": "Hi, I'm your assistant, can I help you?",
-                "role": "assistant"
-            }
-        ],
-        "reference": [],
-        "tokens": 0,
-        "update_date": "Fri, 12 Apr 2024 17:26:21 GMT",
-        "update_time": 1712913981857,
-        "user_id": "kevinhu"
-    },
-    "retcode": 0,
-    "retmsg": "success"
-}
-```
-> data['id'] in response should be stored and will be used in every round of following conversation.
-
-## Get history of a conversation
-
-### Path: /api/conversation/\<id\>
-### Method: GET
-### Response 
-```json
-{
-    "data": {
-        "create_date": "Mon, 01 Apr 2024 09:28:42 GMT",
-        "create_time": 1711934922220,
-        "dialog_id": "df4a4916d7bd11eeaa650242ac180006",
-        "id": "2cae30fcefc711ee94140242ac180006",
-        "message": [
-            {
-                "content": "Hi! I'm your assistant, what can I do for you?",
-                "role": "assistant"
-            },
-            {
-                "content": "What's the vit score for GPT-4?",
-                "role": "user"
-            },
-            {
-                "content": "The ViT Score for GPT-4 in the zero-shot scenario is 0.5058, and in the few-shot scenario, it is 0.6480. ##0$$",
-                "role": "assistant"
-            },
-            {
-                "content": "How is the nvlink topology like?",
-                "role": "user"
-            },
-            {
-                "content": "NVLink topology refers to the arrangement of connections between GPUs using NVIDIA's NVLink technology. Correct NVLink topology for NVIDIA A100 cards involves connecting one GPU to another through a series of NVLink bridges ##0$$. Each of the three attached bridges spans two PCIe slots, and for optimal performance and balanced bridge topology, all three NVLink bridges should be used when connecting two adjacent A100 cards.\n\nHere's a summary of the correct and incorrect topologies:\n\n- **Correct**: Both GPUs are connected via all three NVLink bridges, ensuring full bandwidth and proper communication.\n- **Incorrect**: Not using all three bridges or having an uneven connection configuration would result in suboptimal performance.\n\nIt's also important to note that for multi-CPU systems, both A100 cards in a bridged pair should be within the same CPU domain, unless each CPU has a single A100 PCIe card, in which case they can be bridged together.",
-                "role": "assistant"
-            }
-        ],
-        "user_id": "user name",
-        "reference": [
-            {
-                "chunks": [
-                    {
-                        "chunk_id": "d0bc7892c3ec4aeac071544fd56730a8",
-                        "content_ltks": "tabl 1:openagi task-solv perform under differ set for three closed-sourc llm . boldfac denot the highest score under each learn schema . metric gpt-3.5-turbo claude-2 gpt-4 zero few zero few zero few clip score 0.0 0.0 0.0 0.2543 0.0 0.3055 bert score 0.1914 0.3820 0.2111 0.5038 0.2076 0.6307 vit score 0.2437 0.7497 0.4082 0.5416 0.5058 0.6480 overal 0.1450 0.3772 0.2064 0.4332 0.2378 0.5281",
-                        "content_with_weight": "<table><caption>Table 1: OpenAGI task-solving performances under different settings for three closed-source LLMs. Boldface denotes the highest score under each learning schema.</caption>\n<tr><th  rowspan=2 >Metrics</th><th  >GPT-3.5-turbo</th><th></th><th  >Claude-2</th><th  >GPT-4</th></tr>\n<tr><th  >Zero</th><th  >Few</th><th  >Zero Few</th><th  >Zero Few</th></tr>\n<tr><td  >CLIP Score</td><td  >0.0</td><td  >0.0</td><td  >0.0 0.2543</td><td  >0.0 0.3055</td></tr>\n<tr><td  >BERT Score</td><td  >0.1914</td><td  >0.3820</td><td  >0.2111 0.5038</td><td  >0.2076 0.6307</td></tr>\n<tr><td  >ViT Score</td><td  >0.2437</td><td  >0.7497</td><td  >0.4082 0.5416</td><td  >0.5058 0.6480</td></tr>\n<tr><td  >Overall</td><td  >0.1450</td><td  >0.3772</td><td  >0.2064 0.4332</td><td  >0.2378 0.5281</td></tr>\n</table>",
-                        "doc_id": "c790da40ea8911ee928e0242ac180005",
-                        "docnm_kwd": "OpenAGI When LLM Meets Domain Experts.pdf",
-                        "img_id": "afab9fdad6e511eebdb20242ac180006-d0bc7892c3ec4aeac071544fd56730a8",
-                        "important_kwd": [],
-                        "kb_id": "afab9fdad6e511eebdb20242ac180006",
-                        "positions": [
-                            [
-                                9.0,
-                                159.9383341471354,
-                                472.1773274739583,
-                                223.58013916015625,
-                                307.86692301432294
-                            ]
-                        ],
-                        "similarity": 0.7310340654129031,
-                        "term_similarity": 0.7671974387781668,
-                        "vector_similarity": 0.40556370512552886
-                    },
-                    {
-                        "chunk_id": "7e2345d440383b756670e1b0f43a7007",
-                        "content_ltks": "5.5 experiment analysi the main experiment result are tabul in tab . 1 and 2 , showcas the result for closed-sourc and open-sourc llm , respect . the overal perform is calcul a the averag of cllp 8 bert and vit score . here , onli the task descript of the benchmark task are fed into llm(addit inform , such a the input prompt and llm\u2019output , is provid in fig . a.4 and a.5 in supplementari). broadli speak , closed-sourc llm demonstr superior perform on openagi task , with gpt-4 lead the pack under both zero-and few-shot scenario . in the open-sourc categori , llama-2-13b take the lead , consist post top result across variou learn schema--the perform possibl influenc by it larger model size . notabl , open-sourc llm significantli benefit from the tune method , particularli fine-tun and\u2019rltf . these method mark notic enhanc for flan-t5-larg , vicuna-7b , and llama-2-13b when compar with zero-shot and few-shot learn schema . in fact , each of these open-sourc model hit it pinnacl under the rltf approach . conclus , with rltf tune , the perform of llama-2-13b approach that of gpt-3.5 , illustr it potenti .",
-                        "content_with_weight": "5.5 Experimental Analysis\nThe main experimental results are tabulated in Tab. 1 and 2, showcasing the results for closed-source and open-source LLMs, respectively. The overall performance is calculated as the average of CLlP\n8\nBERT and ViT scores. Here, only the task descriptions of the benchmark tasks are fed into LLMs (additional information, such as the input prompt and LLMs\u2019 outputs, is provided in Fig. A.4 and A.5 in supplementary). Broadly speaking, closed-source LLMs demonstrate superior performance on OpenAGI tasks, with GPT-4 leading the pack under both zero- and few-shot scenarios. In the open-source category, LLaMA-2-13B takes the lead, consistently posting top results across various learning schema--the performance possibly influenced by its larger model size. Notably, open-source LLMs significantly benefit from the tuning methods, particularly Fine-tuning and\u2019 RLTF. These methods mark noticeable enhancements for Flan-T5-Large, Vicuna-7B, and LLaMA-2-13B when compared with zero-shot and few-shot learning schema. In fact, each of these open-source models hits its pinnacle under the RLTF approach. Conclusively, with RLTF tuning, the performance of LLaMA-2-13B approaches that of GPT-3.5, illustrating its potential.",
-                        "doc_id": "c790da40ea8911ee928e0242ac180005",
-                        "docnm_kwd": "OpenAGI When LLM Meets Domain Experts.pdf",
-                        "img_id": "afab9fdad6e511eebdb20242ac180006-7e2345d440383b756670e1b0f43a7007",
-                        "important_kwd": [],
-                        "kb_id": "afab9fdad6e511eebdb20242ac180006",
-                        "positions": [
-                            [
-                                8.0,
-                                107.3,
-                                508.90000000000003,
-                                686.3,
-                                697.0
-                            ],
-                        ],
-                        "similarity": 0.6691508616357027,
-                        "term_similarity": 0.6999011754270821,
-                        "vector_similarity": 0.39239803751328806
-                    },
-                ],
-                "doc_aggs": [
-                    {
-                        "count": 8,
-                        "doc_id": "c790da40ea8911ee928e0242ac180005",
-                        "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf"
-                    }
-                ],
-                "total": 8
-            },
-            {
-                "chunks": [
-                    {
-                        "chunk_id": "8c11a1edddb21ad2ae0c43b4a5dcfa62",
-                        "content_ltks": "nvlink bridg support nvidia\u00aenvlink\u00aei a high-spe point-to-point peer transfer connect , where one gpu can transfer data to and receiv data from one other gpu . the nvidia a100 card support nvlink bridg connect with a singl adjac a100 card . each of the three attach bridg span two pcie slot . to function correctli a well a to provid peak bridg bandwidth , bridg connect with an adjac a100 card must incorpor all three nvlink bridg . wherev an adjac pair of a100 card exist in the server , for best bridg perform and balanc bridg topolog , the a100 pair should be bridg . figur 4 illustr correct and incorrect a100 nvlink connect topolog . nvlink topolog\u2013top view figur 4. correct incorrect correct incorrect for system that featur multipl cpu , both a100 card of a bridg card pair should be within the same cpu domain\u2014that is , under the same cpu\u2019s topolog . ensur thi benefit workload applic perform . the onli except is for dual cpu system wherein each cpu ha a singl a100 pcie card under it;in that case , the two a100 pcie card in the system may be bridg togeth . a100 nvlink speed and bandwidth are given in the follow tabl . tabl 5. a100 nvlink speed and bandwidth paramet valu total nvlink bridg support by nvidia a100 3 total nvlink rx and tx lane support 96 data rate per nvidia a100 nvlink lane(each direct)50 gbp total maximum nvlink bandwidth 600 gbyte per second pb-10137-001_v03|8 nvidia a100 40gb pcie gpu acceler",
-                        "content_with_weight": "NVLink Bridge Support\nNVIDIA\u00aeNVLink\u00aeis a high-speed point-to-point peer transfer connection, where one GPU can transfer data to and receive data from one other GPU. The NVIDIA A100 card supports NVLink bridge connection with a single adjacent A100 card.\nEach of the three attached bridges spans two PCIe slots. To function correctly as well as to provide peak bridge bandwidth, bridge connection with an adjacent A100 card must incorporate all three NVLink bridges. Wherever an adjacent pair of A100 cards exists in the server, for best bridging performance and balanced bridge topology, the A100 pair should be bridged. Figure 4 illustrates correct and incorrect A100 NVLink connection topologies.\nNVLink Topology \u2013Top Views \nFigure 4. \nCORRECT \nINCORRECT \nCORRECT \nINCORRECT \nFor systems that feature multiple CPUs, both A100 cards of a bridged card pair should be within the same CPU domain\u2014that is, under the same CPU\u2019s topology. Ensuring this benefits workload application performance. The only exception is for dual CPU systems wherein each CPU has a single A100 PCIe card under it; in that case, the two A100 PCIe cards in the system may be bridged together.\nA100 NVLink speed and bandwidth are given in the following table.\n<table><caption>Table 5. A100 NVLink Speed and Bandwidth </caption>\n<tr><th  >Parameter </th><th  >Value </th></tr>\n<tr><td  >Total NVLink bridges supported by NVIDIA A100 </td><td  >3 </td></tr>\n<tr><td  >Total NVLink Rx and Tx lanes supported </td><td  >96 </td></tr>\n<tr><td  >Data rate per NVIDIA A100 NVLink lane (each direction)</td><td  >50 Gbps </td></tr>\n<tr><td  >Total maximum NVLink bandwidth</td><td  >600 Gbytes per second </td></tr>\n</table>\nPB-10137-001_v03 |8\nNVIDIA A100 40GB PCIe GPU Accelerator",
-                        "doc_id": "806d1ed0ea9311ee860a0242ac180005",
-                        "docnm_kwd": "A100-PCIE-Prduct-Brief.pdf",
-                        "img_id": "afab9fdad6e511eebdb20242ac180006-8c11a1edddb21ad2ae0c43b4a5dcfa62",
-                        "important_kwd": [],
-                        "kb_id": "afab9fdad6e511eebdb20242ac180006",
-                        "positions": [
-                            [
-                                12.0,
-                                84.0,
-                                541.3,
-                                76.7,
-                                96.7
-                            ],
-                        ],
-                        "similarity": 0.3200748779905588,
-                        "term_similarity": 0.3082244010114718,
-                        "vector_similarity": 0.42672917080234146
-                    },
-                ],
-                "doc_aggs": [
-                    {
-                        "count": 1,
-                        "doc_id": "806d1ed0ea9311ee860a0242ac180005",
-                        "doc_name": "A100-PCIE-Prduct-Brief.pdf"
-                    }
-                ],
-                "total": 3
-            }
-        ],
-        "update_date": "Tue, 02 Apr 2024 09:07:49 GMT",
-        "update_time": 1712020069421
-    },
-    "retcode": 0,
-    "retmsg": "success"
-}
-```
-
-- **message**: All the chat history in it.
-    - role: user or assistant
-    - content: the text content of user or assistant. The citations are in format like: ##0$$. The number in the middle indicate which part in data.reference.chunks it refers to.
-    
-- **user_id**: This is set by the caller.
-- **reference**: Every item in it refer to the corresponding message in data.message whose role is assistant. 
-    - chunks
-        - content_with_weight: The content of chunk.
-        - docnm_kwd: the document name.
-        - img_id: the image id of the chunk. It is an optional field only for PDF/pptx/picture. And accessed by 'GET' /document/get/\<id\>.
-        - positions: [page_number, [upleft corner(x, y)], [right bottom(x, y)]], the chunk position, only for PDF.
-        - similarity: the hybrid similarity.
-        - term_similarity: keyword simimlarity
-        - vector_similarity: embedding similarity
-    - doc_aggs:
-        - doc_id: the document can be accessed by 'GET' /document/get/\<id\>
-        - doc_name: the file name
-        - count: the chunk number hit in this document.
-    
-## Chat
-
-This will be called to get the answer to users' questions.
-
-### Path: /api/completion
-### Method: POST
-### Parameter:
-
-| name | type | optional | description|
-|------|-------|----|----|
-| conversation_id| string | No | This is from calling /new_conversation.|
-| messages| json | No | The latest question, such as `[{"role": "user", "content": "How are you doing!"}]`|
-| quote | bool | Yes | Default: true |
-| stream | bool | Yes | Default: true |
-| doc_ids | string | Yes | Document IDs which is delimited by comma, like `c790da40ea8911ee928e0242ac180005,c790da40ea8911ee928e0242ac180005`. The retrieved content is limited in these documents. |
-
-### Response 
-```json
-{
-    "data": {
-      "answer": "The ViT Score for GPT-4 in the zero-shot scenario is 0.5058, and in the few-shot scenario, it is 0.6480. ##0$$",
-      "reference": {
-        "chunks": [
-          {
-            "chunk_id": "d0bc7892c3ec4aeac071544fd56730a8",
-            "content_ltks": "tabl 1:openagi task-solv perform under differ set for three closed-sourc llm . boldfac denot the highest score under each learn schema . metric gpt-3.5-turbo claude-2 gpt-4 zero few zero few zero few clip score 0.0 0.0 0.0 0.2543 0.0 0.3055 bert score 0.1914 0.3820 0.2111 0.5038 0.2076 0.6307 vit score 0.2437 0.7497 0.4082 0.5416 0.5058 0.6480 overal 0.1450 0.3772 0.2064 0.4332 0.2378 0.5281",
-            "content_with_weight": "<table><caption>Table 1: OpenAGI task-solving performances under different settings for three closed-source LLMs. Boldface denotes the highest score under each learning schema.</caption>\n<tr><th  rowspan=2 >Metrics</th><th  >GPT-3.5-turbo</th><th></th><th  >Claude-2</th><th  >GPT-4</th></tr>\n<tr><th  >Zero</th><th  >Few</th><th  >Zero Few</th><th  >Zero Few</th></tr>\n<tr><td  >CLIP Score</td><td  >0.0</td><td  >0.0</td><td  >0.0 0.2543</td><td  >0.0 0.3055</td></tr>\n<tr><td  >BERT Score</td><td  >0.1914</td><td  >0.3820</td><td  >0.2111 0.5038</td><td  >0.2076 0.6307</td></tr>\n<tr><td  >ViT Score</td><td  >0.2437</td><td  >0.7497</td><td  >0.4082 0.5416</td><td  >0.5058 0.6480</td></tr>\n<tr><td  >Overall</td><td  >0.1450</td><td  >0.3772</td><td  >0.2064 0.4332</td><td  >0.2378 0.5281</td></tr>\n</table>",
-            "doc_id": "c790da40ea8911ee928e0242ac180005",
-            "docnm_kwd": "OpenAGI When LLM Meets Domain Experts.pdf",
-            "img_id": "afab9fdad6e511eebdb20242ac180006-d0bc7892c3ec4aeac071544fd56730a8",
-            "important_kwd": [],
-            "kb_id": "afab9fdad6e511eebdb20242ac180006",
-            "positions": [
-              [
-                9.0,
-                159.9383341471354,
-                472.1773274739583,
-                223.58013916015625,
-                307.86692301432294
-              ]
-            ],
-            "similarity": 0.7310340654129031,
-            "term_similarity": 0.7671974387781668,
-            "vector_similarity": 0.40556370512552886
-          },
-          {
-            "chunk_id": "7e2345d440383b756670e1b0f43a7007",
-            "content_ltks": "5.5 experiment analysi the main experiment result are tabul in tab . 1 and 2 , showcas the result for closed-sourc and open-sourc llm , respect . the overal perform is calcul a the averag of cllp 8 bert and vit score . here , onli the task descript of the benchmark task are fed into llm(addit inform , such a the input prompt and llm\u2019output , is provid in fig . a.4 and a.5 in supplementari). broadli speak , closed-sourc llm demonstr superior perform on openagi task , with gpt-4 lead the pack under both zero-and few-shot scenario . in the open-sourc categori , llama-2-13b take the lead , consist post top result across variou learn schema--the perform possibl influenc by it larger model size . notabl , open-sourc llm significantli benefit from the tune method , particularli fine-tun and\u2019rltf . these method mark notic enhanc for flan-t5-larg , vicuna-7b , and llama-2-13b when compar with zero-shot and few-shot learn schema . in fact , each of these open-sourc model hit it pinnacl under the rltf approach . conclus , with rltf tune , the perform of llama-2-13b approach that of gpt-3.5 , illustr it potenti .",
-            "content_with_weight": "5.5 Experimental Analysis\nThe main experimental results are tabulated in Tab. 1 and 2, showcasing the results for closed-source and open-source LLMs, respectively. The overall performance is calculated as the average of CLlP\n8\nBERT and ViT scores. Here, only the task descriptions of the benchmark tasks are fed into LLMs (additional information, such as the input prompt and LLMs\u2019 outputs, is provided in Fig. A.4 and A.5 in supplementary). Broadly speaking, closed-source LLMs demonstrate superior performance on OpenAGI tasks, with GPT-4 leading the pack under both zero- and few-shot scenarios. In the open-source category, LLaMA-2-13B takes the lead, consistently posting top results across various learning schema--the performance possibly influenced by its larger model size. Notably, open-source LLMs significantly benefit from the tuning methods, particularly Fine-tuning and\u2019 RLTF. These methods mark noticeable enhancements for Flan-T5-Large, Vicuna-7B, and LLaMA-2-13B when compared with zero-shot and few-shot learning schema. In fact, each of these open-source models hits its pinnacle under the RLTF approach. Conclusively, with RLTF tuning, the performance of LLaMA-2-13B approaches that of GPT-3.5, illustrating its potential.",
-            "doc_id": "c790da40ea8911ee928e0242ac180005",
-            "docnm_kwd": "OpenAGI When LLM Meets Domain Experts.pdf",
-            "img_id": "afab9fdad6e511eebdb20242ac180006-7e2345d440383b756670e1b0f43a7007",
-            "important_kwd": [],
-            "kb_id": "afab9fdad6e511eebdb20242ac180006",
-            "positions": [
-              [
-                8.0,
-                107.3,
-                508.90000000000003,
-                686.3,
-                697.0
-              ]
-            ],
-            "similarity": 0.6691508616357027,
-            "term_similarity": 0.6999011754270821,
-            "vector_similarity": 0.39239803751328806
-          }
-        ],
-        "doc_aggs": {
-          "OpenAGI When LLM Meets Domain Experts.pdf": 4
-        },
-        "total": 8
-      }
-    },
-    "retcode": 0,
-    "retmsg": "success"
-}
-```
-
-- **answer**: The replay of the chat bot.
-- **reference**: 
-    - chunks: Every item in it refer to the corresponding message in answer. 
-        - content_with_weight: The content of chunk.
-        - docnm_kwd: the document name.
-        - img_id: the image id of the chunk. It is an optional field only for PDF/pptx/picture. And accessed by 'GET' /document/get/\<id\>.
-        - positions: [page_number, [upleft corner(x, y)], [right bottom(x, y)]], the chunk position, only for PDF.
-        - similarity: the hybrid similarity.
-        - term_similarity: keyword simimlarity
-        - vector_similarity: embedding similarity
-    - doc_aggs:
-        - doc_id: the document can be accessed by 'GET' /document/get/\<id\>
-        - doc_name: the file name
-        - count: the chunk number hit in this document.
-    
-## Get document content or image
-
-This is usually used when display content of citation.
-### Path: /api/document/get/\<id\>
-### Method: GET
-
-## Upload file
-
-This is usually used when upload a file to.
-### Path: /api/document/upload/
-### Method: POST
-
-### Parameter:
-
-| name      | type   | optional | description                                             |
-|-----------|--------|----------|---------------------------------------------------------|
-| file      | file   | No       | Upload file.                                            |
-| kb_name   | string | No       | Choose the upload knowledge base name.                  |
-| parser_id | string | Yes      | Choose the parsing method.                              |
-| run       | string | Yes      | Parsing will start automatically when the value is "1". |
-
-### Response 
-```json
-{
-    "data": {
-        "chunk_num": 0,
-        "create_date": "Thu, 25 Apr 2024 14:30:06 GMT",
-        "create_time": 1714026606921,
-        "created_by": "553ec818fd5711ee8ea63043d7ed348e",
-        "id": "41e9324602cd11ef9f5f3043d7ed348e",
-        "kb_id": "06802686c0a311ee85d6246e9694c130",
-        "location": "readme.txt",
-        "name": "readme.txt",
-        "parser_config": {
-            "field_map": {
-            },
-            "pages": [
-                [
-                    0,
-                    1000000
-                ]
-            ]
-        },
-        "parser_id": "general",
-        "process_begin_at": null,
-        "process_duation": 0.0,
-        "progress": 0.0,
-        "progress_msg": "",
-        "run": "0",
-        "size": 929,
-        "source_type": "local",
-        "status": "1",
-        "thumbnail": null,
-        "token_num": 0,
-        "type": "doc",
-        "update_date": "Thu, 25 Apr 2024 14:30:06 GMT",
-        "update_time": 1714026606921
-    },
-    "retcode": 0,
-    "retmsg": "success"
-}
-
-```
-
-## Get document chunks
-
-Get the chunks of the document based on doc_name or doc_id.
-### Path: /api/list_chunks/
-### Method: POST
-
-### Parameter:
-
-| Name     | Type   | Optional | Description                     |
-|----------|--------|----------|---------------------------------|
-| `doc_name` | string | Yes      | The name of the document in the knowledge base. It must not be empty if `doc_id` is not set.|
-| `doc_id`   | string | Yes      | The ID of the document in the knowledge base. It must not be empty if `doc_name` is not set.|
-
-
-### Response 
-```json
-{
-    "data": [
-        {
-            "content": "Figure 14: Per-request neural-net processingof RL-Cache.\n103\n(sn)\nCPU\n 102\nGPU\n8101\n100\n8\n16 64 256 1K\n4K",
-            "doc_name": "RL-Cache.pdf",
-            "img_id": "0335167613f011ef91240242ac120006-b46c3524952f82dbe061ce9b123f2211"
-        },
-        {
-            "content": "4.3 ProcessingOverheadof RL-CacheACKNOWLEDGMENTSThis section evaluates how e￿ectively our RL-Cache implemen-tation leverages modern multi-core CPUs and GPUs to keep the per-request neural-net processing overhead low. Figure 14 depictsThis researchwas supported inpart by the Regional Government of Madrid (grant P2018/TCS-4499, EdgeData-CM)andU.S. National Science Foundation (grants CNS-1763617 andCNS-1717179).REFERENCES",
-            "doc_name": "RL-Cache.pdf",
-            "img_id": "0335167613f011ef91240242ac120006-d4c12c43938eb55d2d8278eea0d7e6d7"
-        }
-    ],
-    "retcode": 0,
-    "retmsg": "success"
-}
-
-```

docs/guides/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "User Guides",
+  "position": 2,
+  "link": {
+    "type": "generated-index",
+    "description": "RAGFlow User Guides"
+  }
+}

docs/guides/configure_knowledge_base.md

@@ -1,3 +1,8 @@
+---
+sidebar_position: 1
+slug: /configure_knowledge_base
+---
+
 # Configure a knowledge base
 
 Knowledge base, hallucination-free chat, and file management are three pillars of RAGFlow. RAGFlow's AI chats are based on knowledge bases. Each of RAGFlow's knowledge bases serves as a knowledge source, *parsing* files uploaded from your local machine and file references generated in **File Management** into the real 'knowledge' for future AI chats. This guide demonstrates some basic usages of the knowledge base feature, covering the following topics:
@@ -57,6 +62,7 @@ An embedding model builds vector index on file chunks. Once you have chosen an e
 
 The following embedding models can be deployed locally:
 
+- BAAI/bge-large-zh-v1.5
 - BAAI/bge-base-en-v1.5
 - BAAI/bge-large-en-v1.5
 - BAAI/bge-small-en-v1.5
@@ -118,7 +124,7 @@ RAGFlow uses multiple recall of both full-text search and vector search in its c
 
 ## Search for knowledge base
 
-As of RAGFlow v0.5.0, the search feature is still in a rudimentary form, supporting only knowledge base search by name.
+As of RAGFlow v0.7.0, the search feature is still in a rudimentary form, supporting only knowledge base search by name.
 
 ![search knowledge base](https://github.com/infiniflow/ragflow/assets/93570324/836ae94c-2438-42be-879e-c7ad2a59693e)
 

docs/guides/deploy_local_llm.md

@@ -0,0 +1,75 @@
+---
+sidebar_position: 5
+slug: /deploy_local_llm
+---
+
+# Deploy a local LLM
+
+RAGFlow supports deploying LLMs locally using Ollama or Xinference.
+
+## Ollama
+
+One-click deployment of local LLMs, that is [Ollama](https://github.com/ollama/ollama).
+
+### Install
+
+- [Ollama on Linux](https://github.com/ollama/ollama/blob/main/docs/linux.md)
+- [Ollama Windows Preview](https://github.com/ollama/ollama/blob/main/docs/windows.md)
+- [Docker](https://hub.docker.com/r/ollama/ollama)
+
+### Launch Ollama
+
+Decide which LLM you want to deploy ([here's a list for supported LLM](https://ollama.com/library)), say, **mistral**:
+```bash
+$ ollama run mistral
+```
+Or,
+```bash
+$ docker exec -it ollama ollama run mistral
+```
+
+### Use Ollama in RAGFlow
+
+- Go to 'Settings > Model Providers > Models to be added > Ollama'.
+    
+![](https://github.com/infiniflow/ragflow/assets/12318111/a9df198a-226d-4f30-b8d7-829f00256d46)
+
+> Base URL: Enter the base URL where the Ollama service is accessible, like, `http://<your-ollama-endpoint-domain>:11434`.
+
+- Use Ollama Models.
+
+![](https://github.com/infiniflow/ragflow/assets/12318111/60ff384e-5013-41ff-a573-9a543d237fd3)
+
+## Xinference
+
+Xorbits Inference([Xinference](https://github.com/xorbitsai/inference)) empowers you to unleash the full potential of cutting-edge AI models. 
+
+### Install
+
+- [pip install "xinference[all]"](https://inference.readthedocs.io/en/latest/getting_started/installation.html)
+- [Docker](https://inference.readthedocs.io/en/latest/getting_started/using_docker_image.html)
+
+To start a local instance of Xinference, run the following command:
+```bash
+$ xinference-local --host 0.0.0.0 --port 9997
+```
+### Launch Xinference
+
+Decide which LLM you want to deploy ([here's a list for supported LLM](https://inference.readthedocs.io/en/latest/models/builtin/)), say, **mistral**.
+Execute the following command to launch the model, remember to replace `${quantization}` with your chosen quantization method from the options listed above:
+```bash
+$ xinference launch -u mistral --model-name mistral-v0.1 --size-in-billions 7 --model-format pytorch --quantization ${quantization}
+```
+
+### Use Xinference in RAGFlow
+
+- Go to 'Settings > Model Providers > Models to be added > Xinference'.
+    
+![](https://github.com/infiniflow/ragflow/assets/12318111/bcbf4d7a-ade6-44c7-ad5f-0a92c8a73789)
+
+> Base URL: Enter the base URL where the Xinference service is accessible, like, `http://<your-xinference-endpoint-domain>:9997/v1`.
+
+- Use Xinference Models.
+
+![](https://github.com/infiniflow/ragflow/assets/12318111/b01fcb6f-47c9-4777-82e0-f1e947ed615a)
+![](https://github.com/infiniflow/ragflow/assets/12318111/1763dcd1-044f-438d-badd-9729f5b3a144)

docs/guides/llm_api_key_setup.md

@@ -0,0 +1,30 @@
+---
+sidebar_position: 4
+slug: /llm_api_key_setup
+---
+
+# Set your LLM API key
+
+You have two ways to input your LLM API key. 
+
+## Before Starting The System
+
+In **user_default_llm** of [service_conf.yaml](https://github.com/infiniflow/ragflow/blob/main/docker/service_conf.yaml), you need to specify LLM factory and your own _API_KEY_. 
+RAGFlow supports the flowing LLMs, with more coming in the pipeline:
+
+- [OpenAI](https://platform.openai.com/login?launch)
+- [Tongyi-Qianwen](https://dashscope.console.aliyun.com/model), 
+- [ZHIPU-AI](https://open.bigmodel.cn/), 
+- [Moonshot](https://platform.moonshot.cn/docs)
+- [DeepSeek](https://platform.deepseek.com/api-docs/)
+- [Baichuan](https://www.baichuan-ai.com/home)
+- [VolcEngine](https://www.volcengine.com/docs/82379)
+
+After sign in these LLM suppliers, create your own API-Key, they all have a certain amount of free quota.
+
+## After Starting The System
+
+You can also set API-Key in **User Setting** as following:
+
+![](https://github.com/infiniflow/ragflow/assets/12318111/e4e4066c-e964-45ff-bd56-c3fc7fb18bd3)
+

docs/guides/manage_files.md

@@ -1,3 +1,8 @@
+---
+sidebar_position: 3
+slug: /manage_files
+---
+
 #  Manage files
 
 Knowledge base, hallucination-free chat, and file management are three pillars of RAGFlow. RAGFlow's file management allows you to upload files individually or in bulk. You can then link an uploaded file to multiple target knowledge bases. This guide showcases some basic usages of the file management feature. 
@@ -40,11 +45,11 @@ You can link your file to one knowledge base or multiple knowledge bases at one
 
 ## Move file to specified folder
 
-As of RAGFlow v0.5.0, this feature is *not* available.
+As of RAGFlow v0.7.0, this feature is *not* available.
 
 ## Search files or folders
 
-As of RAGFlow v0.5.0, the search feature is still in a rudimentary form, supporting only file and folder search in the current directory by name (files or folders in the child directory will not be retrieved).
+As of RAGFlow v0.7.0, the search feature is still in a rudimentary form, supporting only file and folder search in the current directory by name (files or folders in the child directory will not be retrieved).
 
 ![search file](https://github.com/infiniflow/ragflow/assets/93570324/77ffc2e5-bd80-4ed1-841f-068e664efffe)
 
@@ -76,4 +81,4 @@ RAGFlow's file management allows you to download an uploaded file:
 
 ![download_file](https://github.com/infiniflow/ragflow/assets/93570324/cf3b297f-7d9b-4522-bf5f-4f45743e4ed5)
 
-> As of RAGFlow v0.5.0, bulk download is not supported, nor can you download an entire folder. 
+> As of RAGFlow v0.7.0, bulk download is not supported, nor can you download an entire folder. 

docs/guides/max_map_count.md

@@ -1,66 +1,71 @@
-# Set vm.max_map_count to at least 262144
-
-## Linux
-
-To check the value of `vm.max_map_count`:
-
-```bash
-$ sysctl vm.max_map_count
-```
-
-Reset `vm.max_map_count` to a value at least 262144 if it is not.
-
-```bash
-# In this case, we set it to 262144:
-$ sudo sysctl -w vm.max_map_count=262144
-```
-
-This change will be reset after a system reboot. To ensure your change remains permanent, add or update the `vm.max_map_count` value in **/etc/sysctl.conf** accordingly:
-
-```bash
-vm.max_map_count=262144
-```
-
-## Mac
-
-```bash
-$ screen ~/Library/Containers/com.docker.docker/Data/vms/0/tty
-$ sysctl -w vm.max_map_count=262144
-```
-To exit the screen session, type Ctrl a d.
-
-## Windows and macOS with Docker Desktop
-
-The vm.max_map_count setting must be set via docker-machine:
-
-```bash
-$ docker-machine ssh
-$ sudo sysctl -w vm.max_map_count=262144
-```
-
-## Windows with Docker Desktop WSL 2 backend
-
-To manually set it every time you reboot, you must run the following commands in a command prompt or PowerShell window every time you restart Docker:
-
-```bash
-$ wsl -d docker-desktop -u root
-$ sysctl -w vm.max_map_count=262144
-```
-If you are on these versions of WSL and you do not want to have to run those commands every time you restart Docker, you can globally change every WSL distribution with this setting by modifying your %USERPROFILE%\.wslconfig as follows:
-
-```bash
-[wsl2]
-kernelCommandLine = "sysctl.vm.max_map_count=262144"
-```
-This will cause all WSL2 VMs to have that setting assigned when they start.
-
-If you are on Windows 11, or Windows 10 version 22H2 and have installed the Microsoft Store version of WSL, you can modify the /etc/sysctl.conf within the "docker-desktop" WSL distribution, perhaps with commands like this:
-
-```bash
-$ wsl -d docker-desktop -u root
-$ vi /etc/sysctl.conf
-```
-and appending a line which reads:
-```bash
-vm.max_map_count = 262144
+---
+sidebar_position: 7
+slug: /max_map_count
+---
+
+# Update vm.max_map_count
+
+## Linux
+
+To check the value of `vm.max_map_count`:
+
+```bash
+$ sysctl vm.max_map_count
+```
+
+Reset `vm.max_map_count` to a value at least 262144 if it is not.
+
+```bash
+# In this case, we set it to 262144:
+$ sudo sysctl -w vm.max_map_count=262144
+```
+
+This change will be reset after a system reboot. To ensure your change remains permanent, add or update the `vm.max_map_count` value in **/etc/sysctl.conf** accordingly:
+
+```bash
+vm.max_map_count=262144
+```
+
+## Mac
+
+```bash
+$ screen ~/Library/Containers/com.docker.docker/Data/vms/0/tty
+$ sysctl -w vm.max_map_count=262144
+```
+To exit the screen session, type Ctrl a d.
+
+## Windows and macOS with Docker Desktop
+
+The vm.max_map_count setting must be set via docker-machine:
+
+```bash
+$ docker-machine ssh
+$ sudo sysctl -w vm.max_map_count=262144
+```
+
+## Windows with Docker Desktop WSL 2 backend
+
+To manually set it every time you reboot, you must run the following commands in a command prompt or PowerShell window every time you restart Docker:
+
+```bash
+$ wsl -d docker-desktop -u root
+$ sysctl -w vm.max_map_count=262144
+```
+If you are on these versions of WSL and you do not want to have to run those commands every time you restart Docker, you can globally change every WSL distribution with this setting by modifying your %USERPROFILE%\.wslconfig as follows:
+
+```bash
+[wsl2]
+kernelCommandLine = "sysctl.vm.max_map_count=262144"
+```
+This will cause all WSL2 VMs to have that setting assigned when they start.
+
+If you are on Windows 11, or Windows 10 version 22H2 and have installed the Microsoft Store version of WSL, you can modify the /etc/sysctl.conf within the "docker-desktop" WSL distribution, perhaps with commands like this:
+
+```bash
+$ wsl -d docker-desktop -u root
+$ vi /etc/sysctl.conf
+```
+and appending a line which reads:
+```bash
+vm.max_map_count = 262144
 ```

docs/guides/start_chat.md

@@ -1,3 +1,8 @@
+---
+sidebar_position: 2
+slug: /start_chat
+---
+
 # Start an AI chat
 
 Knowledge base, hallucination-free chat, and file management are three pillars of RAGFlow. Chats in RAGFlow are based on a particular knowledge base or multiple knowledge bases. Once you have created your knowledge base and finished file parsing, you can go ahead and start an AI conversation. 

docs/llm_api_key_setup.md

@@ -1,19 +0,0 @@
-
-## Set Before Starting The System
-
-In **user_default_llm** of [service_conf.yaml](./docker/service_conf.yaml), you need to specify LLM factory and your own _API_KEY_. 
-RagFlow supports the flowing LLM factory, and with more coming in the pipeline:
-
-> [OpenAI](https://platform.openai.com/login?launch), [Tongyi-Qianwen](https://dashscope.console.aliyun.com/model), 
-> [ZHIPU-AI](https://open.bigmodel.cn/), [Moonshot](https://platform.moonshot.cn/docs)
-
-After sign in these LLM suppliers, create your own API-Key, they all have a certain amount of free quota.
-
-## After Starting The System
-
-You can also set API-Key in **User Setting** as following:
-
-<div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/e4e4066c-e964-45ff-bd56-c3fc7fb18bd3" width="1000"/>
-</div>
-

docs/ollama.md

@@ -1,40 +0,0 @@
-# Ollama
-
-<div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/2019e7ee-1e8a-412e-9349-11bbf702e549" width="130"/>
-</div>
-
-One-click deployment of local LLMs, that is [Ollama](https://github.com/ollama/ollama).
-
-## Install
-
-- [Ollama on Linux](https://github.com/ollama/ollama/blob/main/docs/linux.md)
-- [Ollama Windows Preview](https://github.com/ollama/ollama/blob/main/docs/windows.md)
-- [Docker](https://hub.docker.com/r/ollama/ollama)
-
-## Launch Ollama
-
-Decide which LLM you want to deploy ([here's a list for supported LLM](https://ollama.com/library)), say, **mistral**:
-```bash
-$ ollama run mistral
-```
-Or,
-```bash
-$ docker exec -it ollama ollama run mistral
-```
-
-## Use Ollama in RAGFlow
-
-- Go to 'Settings > Model Providers > Models to be added > Ollama'.
-    
-<div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/a9df198a-226d-4f30-b8d7-829f00256d46" width="1300"/>
-</div>
-
-> Base URL: Enter the base URL where the Ollama service is accessible, like, `http://<your-ollama-endpoint-domain>:11434`.
-
-- Use Ollama Models.
-
-<div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/60ff384e-5013-41ff-a573-9a543d237fd3" width="530"/>
-</div>

docs/quickstart.md

@@ -1,4 +1,9 @@
-# Quickstart
+---
+sidebar_position: 1
+slug: /
+---
+
+# Quick start
 
 RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. When integrated with LLMs, it is capable of providing truthful question-answering capabilities, backed by well-founded citations from various complex formatted data.
 
@@ -20,7 +25,9 @@ This quick start guide describes a general process from:
 
 ## Start up the server
 
-1. Ensure `vm.max_map_count` >= 262144 ([more](./docs/max_map_count.md)):
+This section provides instructions on setting up the RAGFlow server on Linux. If you are on a different operating system, no worries. Most steps are alike. 
+
+1. Ensure `vm.max_map_count` >= 262144:
 
    > To check the value of `vm.max_map_count`:
    >
@@ -40,6 +47,7 @@ This quick start guide describes a general process from:
    > ```bash
    > vm.max_map_count=262144
    > ```
+   > See [this guide](./guides/max_map_count.md) for instructions on permanently setting `vm.max_map_count` on an operating system other than Linux. 
 
 2. Clone the repo:
 
@@ -49,7 +57,7 @@ This quick start guide describes a general process from:
 
 3. Build the pre-built Docker images and start up the server:
 
-   > Running the following commands automatically downloads the *dev* version RAGFlow Docker image. To download and run a specified Docker version, update `RAGFLOW_VERSION` in **docker/.env** to the intended version, for example `RAGFLOW_VERSION=v0.6.0`, before running the following commands.
+   > Running the following commands automatically downloads the *dev* version RAGFlow Docker image. To download and run a specified Docker version, update `RAGFLOW_VERSION` in **docker/.env** to the intended version, for example `RAGFLOW_VERSION=v0.7.0`, before running the following commands.
 
    ```bash
    $ cd ragflow/docker
@@ -93,8 +101,11 @@ RAGFlow is a RAG engine, and it needs to work with an LLM to offer grounded, hal
 
 - OpenAI
 - Tongyi-Qianwen
+- ZHIPU-AI
 - Moonshot
 - DeepSeek-V2
+- Baichuan
+- VolcEngine
 
 >  RAGFlow also supports deploying LLMs locally using Ollama or Xinference, but this part is not covered in this quick start guide. 
 
@@ -122,7 +133,7 @@ To add and configure an LLM:
 
    ![system model settings](https://github.com/infiniflow/ragflow/assets/93570324/cdcc1da5-4494-44cd-ad5b-1222ed6acc3f)
 
-> Some of the models, such as the image-to-text model **qwen-vl-max**, are subsidiary to a particular LLM. And you may need to update your API key accordingly to use these models. 
+> Some models, such as the image-to-text model **qwen-vl-max**, are subsidiary to a specific LLM. And you may need to update your API key to access these models. 
 
 ## Create your first knowledge base
 

docs/references/CONTRIBUTING.md

@@ -1,52 +1,52 @@
----
-sidebar_position: 0
-slug: /contribution_guidelines
----
-
-# Contribution Guidelines
-
-Thanks for wanting to contribute to RAGFlow. This document offers guidlines and major considerations for submitting your contributions. 
-
-- To report a bug, file a [GitHub issue](https://github.com/infiniflow/ragflow/issues/new/choose) with us.
-- For further questions, you can explore existing discussions or initiate a new one in [Discussions](https://github.com/orgs/infiniflow/discussions).
-
-
-## What you can contribute
-
-The list below mentions some contributions you can make, but it is not a complete list.
-
-- Proposing or implementing new features
-- Fixing a bug
-- Adding test cases or demos
-- Posting a blog or tutorial
-- Updates to existing documents, codes, or annotations.
-- Suggesting more user-friendly error codes
-
-## File a pull request (PR)
-
-### General workflow
-
-1. Fork our GitHub repository.
-2. Clone your fork to your local machine: 
-`git clone git@github.com:<yourname>/ragflow.git`
-3. Create a local branch: 
-`git checkout -b my-branch`
-4. Provide sufficient information in your commit message
-`git commit -m 'Provide sufficient info in your commit message'`
-5. Commit changes to your local branch, and push to GitHub: (include necessary commit message)
-`git push origin my-branch.`
-6. Submit a pull request for review.
-
-### Before filing a PR
-
-- Consider splitting a large PR into multiple smaller, standalone PRs to keep a traceable development history. 
-- Ensure that your PR addresses just one issue, or keep any unrelated changes small.
-- Add test cases when contributing new features. They demonstrate that your code functions correctly and protect against potential issues from future changes.
-### Describing your PR 
-
-- Ensure that your PR title is concise and clear, providing all the required information.
-- Refer to a corresponding GitHub issue in your PR description if applicable. 
-- Include sufficient design details for *breaking changes* or *API changes* in your description.
-
-### Reviewing & merging a PR
+---
+sidebar_position: 0
+slug: /contribution_guidelines
+---
+
+# Contribution guidelines
+
+Thanks for wanting to contribute to RAGFlow. This document offers guidlines and major considerations for submitting your contributions. 
+
+- To report a bug, file a [GitHub issue](https://github.com/infiniflow/ragflow/issues/new/choose) with us.
+- For further questions, you can explore existing discussions or initiate a new one in [Discussions](https://github.com/orgs/infiniflow/discussions).
+
+
+## What you can contribute
+
+The list below mentions some contributions you can make, but it is not a complete list.
+
+- Proposing or implementing new features
+- Fixing a bug
+- Adding test cases or demos
+- Posting a blog or tutorial
+- Updates to existing documents, codes, or annotations.
+- Suggesting more user-friendly error codes
+
+## File a pull request (PR)
+
+### General workflow
+
+1. Fork our GitHub repository.
+2. Clone your fork to your local machine: 
+`git clone git@github.com:<yourname>/ragflow.git`
+3. Create a local branch: 
+`git checkout -b my-branch`
+4. Provide sufficient information in your commit message
+`git commit -m 'Provide sufficient info in your commit message'`
+5. Commit changes to your local branch, and push to GitHub: (include necessary commit message)
+`git push origin my-branch.`
+6. Submit a pull request for review.
+
+### Before filing a PR
+
+- Consider splitting a large PR into multiple smaller, standalone PRs to keep a traceable development history. 
+- Ensure that your PR addresses just one issue, or keep any unrelated changes small.
+- Add test cases when contributing new features. They demonstrate that your code functions correctly and protect against potential issues from future changes.
+### Describing your PR 
+
+- Ensure that your PR title is concise and clear, providing all the required information.
+- Refer to a corresponding GitHub issue in your PR description if applicable. 
+- Include sufficient design details for *breaking changes* or *API changes* in your description.
+
+### Reviewing & merging a PR
 - Ensure that your PR passes all Continuous Integration (CI) tests before merging it.

docs/references/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "References",
+  "position": 3,
+  "link": {
+    "type": "generated-index",
+    "description": "RAGFlow References"
+  }
+}

docs/references/api.md

@@ -0,0 +1,506 @@
+---
+sidebar_position: 1
+slug: /api
+---
+
+# API reference
+
+RAGFlow offers RESTful APIs for you to integrate its capabilities into third-party applications. 
+
+## Base URL
+```
+https://demo.ragflow.io/v1/
+```
+
+## Authorization
+
+All of RAGFlow's RESTFul APIs use API key for authorization, so keep it safe and do not expose it to the front end. 
+Put your API key in the request header. 
+
+```buildoutcfg
+Authorization: Bearer {API_KEY}
+```
+
+To get your API key:
+
+1. In RAGFlow, click **Chat** tab in the middle top of the page.
+2. Hover over the corresponding dialogue **>** **Chat Bot API** to show the chatbot API configuration page.
+3. Click **Api Key** **>** **Create new key** to create your API key.
+4. Copy and keep your API key safe. 
+
+## Create conversation
+
+This method creates (news) a conversation for a specific user. 
+
+### Request
+
+#### Request URI
+
+| Method   |        Request URI                                          |
+|----------|-------------------------------------------------------------|
+| GET      | `/api/new_conversation`                                     |
+
+:::note
+You are *required* to save the `data.id` value returned in the response data, which is the session ID for all upcoming conversations.
+:::
+
+#### Request parameter
+
+| Name     |  Type  | Required |        Description                                          |
+|----------|--------|----------|-------------------------------------------------------------|
+| `user_id`| string |   Yes    | The unique identifier assigned to each user. `user_id` must be less than 32 characters and cannot be empty. The following character sets are supported: <br />- 26 lowercase English letters (a-z)<br />- 26 uppercase English letters (A-Z)<br />- 10 digits (0-9)<br />- "_", "-", "." |
+
+### Response 
+
+```json
+{
+    "data": {
+        "create_date": "Fri, 12 Apr 2024 17:26:21 GMT",
+        "create_time": 1712913981857,
+        "dialog_id": "4f0a2e4cb9af11ee9ba20aef05f5e94f",
+        "duration": 0.0,
+        "id": "b9b2e098f8ae11ee9f45fa163e197198",
+        "message": [
+            {
+                "content": "Hi, I'm your assistant, what can I do for you?",
+                "role": "assistant"
+            }
+        ],
+        "reference": [],
+        "tokens": 0,
+        "update_date": "Fri, 12 Apr 2024 17:26:21 GMT",
+        "update_time": 1712913981857,
+        "user_id": "<USER_ID_SET_BY_THE_CALLER>"
+    },
+    "retcode": 0,
+    "retmsg": "success"
+}
+```
+
+## Get conversation history
+
+This method retrieves the history of a specified conversation session. 
+
+### Request
+
+#### Request URI
+
+| Method   |        Request URI                                          |
+|----------|-------------------------------------------------------------|
+| GET      | `/api/conversation/<id>`                                    |
+
+#### Request parameter
+
+| Name     |  Type  | Required |        Description                                          |
+|----------|--------|----------|-------------------------------------------------------------|
+| `id`     | string |   Yes    | The unique identifier assigned to a conversation session. `id` must be less than 32 characters and cannot be empty. The following character sets are supported: <br />- 26 lowercase English letters (a-z)<br />- 26 uppercase English letters (A-Z)<br />- 10 digits (0-9)<br />- "_", "-", "." |
+
+### Response 
+
+#### Response parameter
+
+- `message`: All conversations in the specified conversation session.
+    - `role`: `"user"` or `"assistant"`.
+    - `content`: The text content of user or assistant. The citations are in a format like `##0$$`. The number in the middle, 0 in this case, indicates which part in data.reference.chunks it refers to.
+    
+- `user_id`: This is set by the caller.
+- `reference`: Each reference corresponds to one of the assistant's answers in `data.message`.
+    - `chunks`
+        - `content_with_weight`: Content of the chunk.
+        - `doc_name`: Name of the *hit* document.
+        - `img_id`: The image ID of the chunk. It is an optional field only for PDF, PPTX, and images. Call ['GET' /document/get/\<id\>](#get-document-content) to retrieve the image.
+        - positions: [page_number, [upleft corner(x, y)], [right bottom(x, y)]], the chunk position, only for PDF.
+        - similarity: The hybrid similarity.
+        - term_similarity: The keyword simimlarity.
+        - vector_similarity: The embedding similarity.
+    - `doc_aggs`:
+        - `doc_id`: ID of the *hit* document. Call ['GET' /document/get/\<id\>](#get-document-content) to retrieve the document.
+        - `doc_name`: Name of the *hit* document.
+        - `count`: The number of *hit* chunks in this document.
+
+```json
+{
+    "data": {
+        "create_date": "Mon, 01 Apr 2024 09:28:42 GMT",
+        "create_time": 1711934922220,
+        "dialog_id": "df4a4916d7bd11eeaa650242ac180006",
+        "id": "2cae30fcefc711ee94140242ac180006",
+        "message": [
+            {
+                "content": "Hi! I'm your assistant, what can I do for you?",
+                "role": "assistant"
+            },
+            {
+                "content": "What's the vit score for GPT-4?",
+                "role": "user"
+            },
+            {
+                "content": "The ViT Score for GPT-4 in the zero-shot scenario is 0.5058, and in the few-shot scenario, it is 0.6480. ##0$$",
+                "role": "assistant"
+            }
+        ],
+        "user_id": "<USER_ID_SET_BY_THE_CALLER>",
+        "reference": [
+            {
+                "chunks": [
+                    {
+                        "chunk_id": "d0bc7892c3ec4aeac071544fd56730a8",
+                        "content_ltks": "tabl 1:openagi task-solv perform under differ set for three closed-sourc llm . boldfac denot the highest score under each learn schema . metric gpt-3.5-turbo claude-2 gpt-4 zero few zero few zero few clip score 0.0 0.0 0.0 0.2543 0.0 0.3055 bert score 0.1914 0.3820 0.2111 0.5038 0.2076 0.6307 vit score 0.2437 0.7497 0.4082 0.5416 0.5058 0.6480 overal 0.1450 0.3772 0.2064 0.4332 0.2378 0.5281",
+                        "content_with_weight": "<table><caption>Table 1: OpenAGI task-solving performances under different settings for three closed-source LLMs. Boldface denotes the highest score under each learning schema.</caption>\n<tr><th  rowspan=2 >Metrics</th><th  >GPT-3.5-turbo</th><th></th><th  >Claude-2</th><th  >GPT-4</th></tr>\n<tr><th  >Zero</th><th  >Few</th><th  >Zero Few</th><th  >Zero Few</th></tr>\n<tr><td  >CLIP Score</td><td  >0.0</td><td  >0.0</td><td  >0.0 0.2543</td><td  >0.0 0.3055</td></tr>\n<tr><td  >BERT Score</td><td  >0.1914</td><td  >0.3820</td><td  >0.2111 0.5038</td><td  >0.2076 0.6307</td></tr>\n<tr><td  >ViT Score</td><td  >0.2437</td><td  >0.7497</td><td  >0.4082 0.5416</td><td  >0.5058 0.6480</td></tr>\n<tr><td  >Overall</td><td  >0.1450</td><td  >0.3772</td><td  >0.2064 0.4332</td><td  >0.2378 0.5281</td></tr>\n</table>",
+                        "doc_id": "c790da40ea8911ee928e0242ac180005",
+                        "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf",
+                        "img_id": "afab9fdad6e511eebdb20242ac180006-d0bc7892c3ec4aeac071544fd56730a8",
+                        "important_kwd": [],
+                        "kb_id": "afab9fdad6e511eebdb20242ac180006",
+                        "positions": [
+                            [
+                                9.0,
+                                159.9383341471354,
+                                472.1773274739583,
+                                223.58013916015625,
+                                307.86692301432294
+                            ]
+                        ],
+                        "similarity": 0.7310340654129031,
+                        "term_similarity": 0.7671974387781668,
+                        "vector_similarity": 0.40556370512552886
+                    },
+                    {
+                        "chunk_id": "7e2345d440383b756670e1b0f43a7007",
+                        "content_ltks": "5.5 experiment analysi the main experiment result are tabul in tab . 1 and 2 , showcas the result for closed-sourc and open-sourc llm , respect . the overal perform is calcul a the averag of cllp 8 bert and vit score . ",
+                        "content_with_weight": "5.5 Experimental Analysis\nThe main experimental results are tabulated in Tab. 1 and 2, showcasing the results for closed-source and open-source LLMs, respectively. The overall performance is calculated as the average of CLlP\n8\nBERT and ViT scores.",
+                        "doc_id": "c790da40ea8911ee928e0242ac180005",
+                        "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf",
+                        "img_id": "afab9fdad6e511eebdb20242ac180006-7e2345d440383b756670e1b0f43a7007",
+                        "important_kwd": [],
+                        "kb_id": "afab9fdad6e511eebdb20242ac180006",
+                        "positions": [
+                            [
+                                8.0,
+                                107.3,
+                                508.90000000000003,
+                                686.3,
+                                697.0
+                            ],
+                        ],
+                        "similarity": 0.6691508616357027,
+                        "term_similarity": 0.6999011754270821,
+                        "vector_similarity": 0.39239803751328806
+                    },
+                ],
+                "doc_aggs": [
+                    {
+                        "count": 8,
+                        "doc_id": "c790da40ea8911ee928e0242ac180005",
+                        "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf"
+                    }
+                ],
+                "total": 8
+            },
+        ],
+        "update_date": "Tue, 02 Apr 2024 09:07:49 GMT",
+        "update_time": 1712020069421
+    },
+    "retcode": 0,
+    "retmsg": "success"
+}
+```
+    
+## Get answer
+
+This method retrieves from RAGFlow the answer to the user's latest question.
+
+### Request
+
+#### Request URI
+
+| Method   |        Request URI                                          |
+|----------|-------------------------------------------------------------|
+| POST     | `/api/completion`                                           |
+
+#### Request parameter
+
+|   Name           |  Type  | Required | Description   |
+|------------------|--------|----------|---------------|
+| `conversation_id`| string | Yes      | The ID of the conversation session. Call ['GET' /new_conversation](#create-conversation) to retrieve the ID.|
+| `messages`       |  json  | Yes      | The latest question in a JSON form, such as `[{"role": "user", "content": "How are you doing!"}]`|
+| `quote`          |  bool  |  No      | Default: true |
+| `stream`         |  bool  |  No      | Default: true |
+| `doc_ids`        | string |  No      | Document IDs delimited by comma, like `c790da40ea8911ee928e0242ac180005,23dsf34ree928e0242ac180005`. The retrieved contents will be confined to these documents. |
+
+### Response 
+
+- `answer`: The answer to the user's latest question.
+- `reference`: 
+    - `chunks`: The retrieved chunks that contribute to the answer.  
+        - `content_with_weight`: Content of the chunk.
+        - `doc_name`: Name of the *hit* document.
+        - `img_id`: The image ID of the chunk. It is an optional field only for PDF, PPTX, and images. Call ['GET' /document/get/\<id\>](#get-document-content) to retrieve the image.
+        - `positions`: [page_number, [upleft corner(x, y)], [right bottom(x, y)]], the chunk position, only for PDF.
+        - `similarity`: The hybrid similarity.
+        - `term_similarity`: The keyword simimlarity.
+        - `vector_similarity`: The embedding similarity.
+    - `doc_aggs`:
+        - `doc_id`: ID of the *hit* document. Call ['GET' /document/get/\<id\>](#get-document-content) to retrieve the document.
+        - `doc_name`: Name of the *hit* document. 
+        - `count`: The number of *hit* chunks in this document.
+
+```json
+{
+    "data": {
+      "answer": "The ViT Score for GPT-4 in the zero-shot scenario is 0.5058, and in the few-shot scenario, it is 0.6480. ##0$$",
+      "reference": {
+        "chunks": [
+          {
+            "chunk_id": "d0bc7892c3ec4aeac071544fd56730a8",
+            "content_ltks": "tabl 1:openagi task-solv perform under differ set for three closed-sourc llm . boldfac denot the highest score under each learn schema . metric gpt-3.5-turbo claude-2 gpt-4 zero few zero few zero few clip score 0.0 0.0 0.0 0.2543 0.0 0.3055 bert score 0.1914 0.3820 0.2111 0.5038 0.2076 0.6307 vit score 0.2437 0.7497 0.4082 0.5416 0.5058 0.6480 overal 0.1450 0.3772 0.2064 0.4332 0.2378 0.5281",
+            "content_with_weight": "<table><caption>Table 1: OpenAGI task-solving performances under different settings for three closed-source LLMs. Boldface denotes the highest score under each learning schema.</caption>\n<tr><th  rowspan=2 >Metrics</th><th  >GPT-3.5-turbo</th><th></th><th  >Claude-2</th><th  >GPT-4</th></tr>\n<tr><th  >Zero</th><th  >Few</th><th  >Zero Few</th><th  >Zero Few</th></tr>\n<tr><td  >CLIP Score</td><td  >0.0</td><td  >0.0</td><td  >0.0 0.2543</td><td  >0.0 0.3055</td></tr>\n<tr><td  >BERT Score</td><td  >0.1914</td><td  >0.3820</td><td  >0.2111 0.5038</td><td  >0.2076 0.6307</td></tr>\n<tr><td  >ViT Score</td><td  >0.2437</td><td  >0.7497</td><td  >0.4082 0.5416</td><td  >0.5058 0.6480</td></tr>\n<tr><td  >Overall</td><td  >0.1450</td><td  >0.3772</td><td  >0.2064 0.4332</td><td  >0.2378 0.5281</td></tr>\n</table>",
+            "doc_id": "c790da40ea8911ee928e0242ac180005",
+            "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf",
+            "img_id": "afab9fdad6e511eebdb20242ac180006-d0bc7892c3ec4aeac071544fd56730a8",
+            "important_kwd": [],
+            "kb_id": "afab9fdad6e511eebdb20242ac180006",
+            "positions": [
+              [
+                9.0,
+                159.9383341471354,
+                472.1773274739583,
+                223.58013916015625,
+                307.86692301432294
+              ]
+            ],
+            "similarity": 0.7310340654129031,
+            "term_similarity": 0.7671974387781668,
+            "vector_similarity": 0.40556370512552886
+          },
+          {
+            "chunk_id": "7e2345d440383b756670e1b0f43a7007",
+            "content_ltks": "5.5 experiment analysi the main experiment result are tabul in tab . 1 and 2 , showcas the result for closed-sourc and open-sourc llm , respect . the overal perform is calcul a the averag of cllp 8 bert and vit score . here , onli the task descript of the benchmark task are fed into llm(addit inform , such a the input prompt and llm\u2019output , is provid in fig . a.4 and a.5 in supplementari). broadli speak , closed-sourc llm demonstr superior perform on openagi task , with gpt-4 lead the pack under both zero-and few-shot scenario . in the open-sourc categori , llama-2-13b take the lead , consist post top result across variou learn schema--the perform possibl influenc by it larger model size . notabl , open-sourc llm significantli benefit from the tune method , particularli fine-tun and\u2019rltf . these method mark notic enhanc for flan-t5-larg , vicuna-7b , and llama-2-13b when compar with zero-shot and few-shot learn schema . in fact , each of these open-sourc model hit it pinnacl under the rltf approach . conclus , with rltf tune , the perform of llama-2-13b approach that of gpt-3.5 , illustr it potenti .",
+            "content_with_weight": "5.5 Experimental Analysis\nThe main experimental results are tabulated in Tab. 1 and 2, showcasing the results for closed-source and open-source LLMs, respectively. The overall performance is calculated as the average of CLlP\n8\nBERT and ViT scores. Here, only the task descriptions of the benchmark tasks are fed into LLMs (additional information, such as the input prompt and LLMs\u2019 outputs, is provided in Fig. A.4 and A.5 in supplementary). Broadly speaking, closed-source LLMs demonstrate superior performance on OpenAGI tasks, with GPT-4 leading the pack under both zero- and few-shot scenarios. In the open-source category, LLaMA-2-13B takes the lead, consistently posting top results across various learning schema--the performance possibly influenced by its larger model size. Notably, open-source LLMs significantly benefit from the tuning methods, particularly Fine-tuning and\u2019 RLTF. These methods mark noticeable enhancements for Flan-T5-Large, Vicuna-7B, and LLaMA-2-13B when compared with zero-shot and few-shot learning schema. In fact, each of these open-source models hits its pinnacle under the RLTF approach. Conclusively, with RLTF tuning, the performance of LLaMA-2-13B approaches that of GPT-3.5, illustrating its potential.",
+            "doc_id": "c790da40ea8911ee928e0242ac180005",
+            "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf",
+            "img_id": "afab9fdad6e511eebdb20242ac180006-7e2345d440383b756670e1b0f43a7007",
+            "important_kwd": [],
+            "kb_id": "afab9fdad6e511eebdb20242ac180006",
+            "positions": [
+              [
+                8.0,
+                107.3,
+                508.90000000000003,
+                686.3,
+                697.0
+              ]
+            ],
+            "similarity": 0.6691508616357027,
+            "term_similarity": 0.6999011754270821,
+            "vector_similarity": 0.39239803751328806
+          }
+        ],
+        "doc_aggs": {
+          "OpenAGI When LLM Meets Domain Experts.pdf": 4
+        },
+        "total": 8
+      }
+    },
+    "retcode": 0,
+    "retmsg": "success"
+}
+```  
+
+## Get document content
+
+This method retrieves the content of a document.
+
+### Request
+
+#### Request URI
+
+| Method   |        Request URI                                          |
+|----------|-------------------------------------------------------------|
+| GET      | `/document/get/<id>`                                        |
+
+### Response
+
+A binary file. 
+
+## Upload file
+
+This method uploads a specific file to a specified knowledge base.
+
+### Request
+
+#### Request URI
+
+| Method   |        Request URI                                          |
+|----------|-------------------------------------------------------------|
+| POST     | `/api/document/upload`                                      |
+
+#### Response parameter
+
+|   Name      | Type   | Required | Description                                             |
+|-------------|--------|----------|---------------------------------------------------------|
+| `file`      | file   | Yes      | The file to upload.                                     |
+| `kb_name`   | string | Yes      | The name of the knowledge base to upload the file to.   |
+| `parser_id` | string |  No      | The parsing method (chunk template) to use. <br />- "naive": General;<br />- "qa": Q&A;<br />- "manual": Manual;<br />- "table": Table;<br />- "paper": Paper;<br />- "laws": Laws;<br />- "presentation": Presentation;<br />- "picture": Picture;<br />- "one": One. |
+| `run`       | string |  No      | 1: Automatically start file parsing. If `parser_id` is not set, RAGFlow uses the general template by default. |
+
+
+### Response 
+
+```json
+{
+    "data": {
+        "chunk_num": 0,
+        "create_date": "Thu, 25 Apr 2024 14:30:06 GMT",
+        "create_time": 1714026606921,
+        "created_by": "553ec818fd5711ee8ea63043d7ed348e",
+        "id": "41e9324602cd11ef9f5f3043d7ed348e",
+        "kb_id": "06802686c0a311ee85d6246e9694c130",
+        "location": "readme.txt",
+        "name": "readme.txt",
+        "parser_config": {
+            "field_map": {
+            },
+            "pages": [
+                [
+                    0,
+                    1000000
+                ]
+            ]
+        },
+        "parser_id": "general",
+        "process_begin_at": null,
+        "process_duation": 0.0,
+        "progress": 0.0,
+        "progress_msg": "",
+        "run": "0",
+        "size": 929,
+        "source_type": "local",
+        "status": "1",
+        "thumbnail": null,
+        "token_num": 0,
+        "type": "doc",
+        "update_date": "Thu, 25 Apr 2024 14:30:06 GMT",
+        "update_time": 1714026606921
+    },
+    "retcode": 0,
+    "retmsg": "success"
+}
+```
+
+## Get document chunks
+
+This method retrieves the chunks of a specific document by `doc_name` or `doc_id`.
+
+### Request
+
+#### Request URI
+
+| Method   |        Request URI                                          |
+|----------|-------------------------------------------------------------|
+| GET      | `/api/list_chunks`                                          |
+
+#### Request parameter
+
+|   Name     | Type   | Required |                        Description                                                          |
+|------------|--------|----------|---------------------------------------------------------------------------------------------|
+| `doc_name` | string |  No      | The name of the document in the knowledge base. It must not be empty if `doc_id` is not set.|
+| `doc_id`   | string |  No      | The ID of the document in the knowledge base. It must not be empty if `doc_name` is not set.|
+
+
+### Response
+
+```json
+{
+    "data": [
+        {
+            "content": "Figure 14: Per-request neural-net processingof RL-Cache.\n103\n(sn)\nCPU\n 102\nGPU\n8101\n100\n8\n16 64 256 1K\n4K",
+            "doc_name": "RL-Cache.pdf",
+            "img_id": "0335167613f011ef91240242ac120006-b46c3524952f82dbe061ce9b123f2211"
+        },
+        {
+            "content": "4.3 ProcessingOverheadof RL-CacheACKNOWLEDGMENTSThis section evaluates how effectively our RL-Cache implemen-tation leverages modern multi-core CPUs and GPUs to keep the per-request neural-net processing overhead low. Figure 14 depictsThis researchwas supported inpart by the Regional Government of Madrid (grant P2018/TCS-4499, EdgeData-CM)andU.S. National Science Foundation (grants CNS-1763617 andCNS-1717179).REFERENCES",
+            "doc_name": "RL-Cache.pdf",
+            "img_id": "0335167613f011ef91240242ac120006-d4c12c43938eb55d2d8278eea0d7e6d7"
+        }
+    ],
+    "retcode": 0,
+    "retmsg": "success"
+}
+```
+
+## Get document list
+
+This method retrieves a list of documents from a specified knowledge base.
+
+### Request
+
+#### Request URI
+
+| Method   |        Request URI                                          |
+|----------|-------------------------------------------------------------|
+| POST     | `/api/list_kb_docs`                                         |
+
+#### Request parameter
+
+| Name        | Type   | Required |  Description                                                          |
+|-------------|--------|----------|-----------------------------------------------------------------------|
+| `kb_name`   | string | Yes      | The name of the knowledge base, from which you get the document list. |
+| `page`      | int    |  No      | The number of pages, default:1.                                       |
+| `page_size` | int    |  No      | The number of docs for each page, default:15.                         |
+| `orderby`   | string |  No      | `chunk_num`, `create_time`, or `size`, default:`create_time`          |
+| `desc`      | bool   |  No      | Default:True.                                                         |
+| `keywords`  | string |  No      | Keyword of the document name.                                         |
+
+
+### Response 
+
+```json
+{
+    "data": {
+        "docs": [
+            {
+                "doc_id": "bad89a84168c11ef9ce40242ac120006",
+                "doc_name": "test.xlsx"
+            },
+            {
+                "doc_id": "641a9b4013f111efb53f0242ac120006",
+                "doc_name": "1111.pdf"
+            }
+        ],
+        "total": 2
+    },
+    "retcode": 0,
+    "retmsg": "success"
+}
+```
+
+## Delete documents 
+
+This method deletes documents by document ID or name.
+
+### Request
+
+#### Request URI
+
+| Method   |        Request URI                                          |
+|----------|-------------------------------------------------------------|
+| DELETE   | `/api/document`                                             |
+
+#### Request parameter
+
+| Name        | Type   | Required | Description                |
+|-------------|--------|----------|----------------------------|
+| `doc_names` | List   |  No      | A list of document names. It must not be empty if `doc_ids` is not set.  |
+| `doc_ids`   | List   |  No      | A list of document IDs. It must not be empty if `doc_names` is not set.  |
+
+
+### Response
+
+```json
+{
+    "data": true,
+    "retcode": 0,
+    "retmsg": "success"
+}
+```

docs/references/faq.md

@@ -1,4 +1,9 @@
-# Frequently Asked Questions
+---
+sidebar_position: 3
+slug: /faq
+---
+
+# Frequently asked questions
 
 ## General
 
@@ -13,6 +18,19 @@ The "garbage in garbage out" status quo remains unchanged despite the fact that
 
 English, simplified Chinese, traditional Chinese for now. 
 
+### 3. Which embedding models can be deployed locally?
+
+- BAAI/bge-large-zh-v1.5
+- BAAI/bge-base-en-v1.5
+- BAAI/bge-large-en-v1.5
+- BAAI/bge-small-en-v1.5
+- BAAI/bge-small-zh-v1.5
+- jinaai/jina-embeddings-v2-base-en
+- jinaai/jina-embeddings-v2-small-en
+- nomic-ai/nomic-embed-text-v1.5
+- sentence-transformers/all-MiniLM-L6-v2
+- maidalun1020/bce-embedding-base_v1
+
 ## Performance
 
 ### 1. Why does it take longer for RAGFlow to parse a document than LangChain?
@@ -31,7 +49,7 @@ Currently, we only support x86 CPU and Nvidia GPU.
 
 ### 2. Do you offer an API for integration with third-party applications?
 
-The corresponding APIs are now available. See the [Conversation API](./conversation_api.md) for more information. 
+The corresponding APIs are now available. See the [RAGFlow API Reference](./api.md) for more information. 
 
 ### 3. Do you support stream output?
 
@@ -186,14 +204,12 @@ Parsing requests have to wait in queue due to limited server resources. We are c
 
 If your RAGFlow is deployed *locally*, try the following: 
 
-1. Click the red cross icon next to **Parsing Status** and refresh the file parsing process. 
-2. If the issue still persists, try the following: 
-   - check the log of your RAGFlow server to see if it is running properly:
-   ```bash
-   docker logs -f ragflow-server
-   ```
-   - Check if the **task_executor.py** process exists.
-   - Check if your RAGFlow server can access hf-mirror.com or huggingface.com.
+1. Check the log of your RAGFlow server to see if it is running properly:
+```bash
+docker logs -f ragflow-server
+```
+2. Check if the **task_executor.py** process exists.
+3. Check if your RAGFlow server can access hf-mirror.com or huggingface.com.
 
 #### 4.5 Why does my pdf parsing stall near completion, while the log does not show any error?
 
@@ -356,7 +372,7 @@ You limit what the system responds to what you specify in **Empty response** if
 
 ### 4. How to run RAGFlow with a locally deployed LLM?
 
-You can use Ollama to deploy local LLM. See [here](https://github.com/infiniflow/ragflow/blob/main/docs/ollama.md) for more information. 
+You can use Ollama to deploy local LLM. See [here](https://github.com/infiniflow/ragflow/blob/main/docs/guides/deploy_local_llm.md) for more information. 
 
 ### 5. How to link up ragflow and ollama servers?
 

docs/xinference.md

@@ -1,43 +0,0 @@
-# Xinference
-
-<div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/2c5e86a7-807b-4d29-bd2b-f73fb1018866" width="130"/>
-</div>
-
-Xorbits Inference([Xinference](https://github.com/xorbitsai/inference)) empowers you to unleash the full potential of cutting-edge AI models. 
-
-## Install
-
-- [pip install "xinference[all]"](https://inference.readthedocs.io/en/latest/getting_started/installation.html)
-- [Docker](https://inference.readthedocs.io/en/latest/getting_started/using_docker_image.html)
-
-To start a local instance of Xinference, run the following command:
-```bash
-$ xinference-local --host 0.0.0.0 --port 9997
-```
-## Launch Xinference
-
-Decide which LLM you want to deploy ([here's a list for supported LLM](https://inference.readthedocs.io/en/latest/models/builtin/)), say, **mistral**.
-Execute the following command to launch the model, remember to replace ${quantization} with your chosen quantization method from the options listed above:
-```bash
-$ xinference launch -u mistral --model-name mistral-v0.1 --size-in-billions 7 --model-format pytorch --quantization ${quantization}
-```
-
-## Use Xinference in RAGFlow
-
-- Go to 'Settings > Model Providers > Models to be added > Xinference'.
-    
-<div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/bcbf4d7a-ade6-44c7-ad5f-0a92c8a73789" width="1300"/>
-</div>
-
-> Base URL: Enter the base URL where the Xinference service is accessible, like, `http://<your-xinference-endpoint-domain>:9997/v1`.
-
-- Use Xinference Models.
-
-<div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/b01fcb6f-47c9-4777-82e0-f1e947ed615a" width="530"/>
-</div>
-<div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/1763dcd1-044f-438d-badd-9729f5b3a144" width="530"/>
-</div>

rag/app/book.py

@@ -19,7 +19,7 @@ from rag.nlp import bullets_category, is_english, tokenize, remove_contents_tabl
     hierarchical_merge, make_colon_as_title, naive_merge, random_choices, tokenize_table, add_positions, \
     tokenize_chunks, find_codec
 from rag.nlp import rag_tokenizer
-from deepdoc.parser import PdfParser, DocxParser, PlainParser
+from deepdoc.parser import PdfParser, DocxParser, PlainParser, HtmlParser
 
 
 class Pdf(PdfParser):
@@ -105,6 +105,14 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
             random_choices([t for t, _ in sections], k=200)))
         callback(0.8, "Finish parsing.")
 
+    elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE):
+        callback(0.1, "Start to parse.")
+        sections = HtmlParser()(filename, binary)
+        sections = [(l, "") for l in sections if l]
+        remove_contents_table(sections, eng=is_english(
+            random_choices([t for t, _ in sections], k=200)))
+        callback(0.8, "Finish parsing.")
+
     elif re.search(r"\.doc$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         binary = BytesIO(binary)
@@ -127,7 +135,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
                   for ck in hierarchical_merge(bull, sections, 5)]
     else:
         sections = [s.split("@") for s, _ in sections]
-        sections = [(pr[0], "@" + pr[1]) for pr in sections if len(pr) == 2]
+        sections = [(pr[0], "@" + pr[1]) if len(pr) == 2 else (pr[0], '') for pr in sections ]
         chunks = naive_merge(
             sections, kwargs.get(
                 "chunk_token_num", 256), kwargs.get(

rag/app/laws.py

@@ -20,7 +20,7 @@ from api.db import ParserType
 from rag.nlp import bullets_category, is_english, tokenize, remove_contents_table, hierarchical_merge, \
     make_colon_as_title, add_positions, tokenize_chunks, find_codec
 from rag.nlp import rag_tokenizer
-from deepdoc.parser import PdfParser, DocxParser, PlainParser
+from deepdoc.parser import PdfParser, DocxParser, PlainParser, HtmlParser
 from rag.settings import cron_logger
 
 
@@ -125,6 +125,12 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         sections = [l for l in sections if l]
         callback(0.8, "Finish parsing.")
 
+    elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE):
+        callback(0.1, "Start to parse.")
+        sections = HtmlParser()(filename, binary)
+        sections = [l for l in sections if l]
+        callback(0.8, "Finish parsing.")
+
     elif re.search(r"\.doc$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         binary = BytesIO(binary)

rag/app/naive.py

@@ -17,8 +17,10 @@ from timeit import default_timer as timer
 import re
 from deepdoc.parser.pdf_parser import PlainParser
 from rag.nlp import rag_tokenizer, naive_merge, tokenize_table, tokenize_chunks, find_codec
-from deepdoc.parser import PdfParser, ExcelParser, DocxParser
+from deepdoc.parser import PdfParser, ExcelParser, DocxParser, HtmlParser
 from rag.settings import cron_logger
+from rag.utils import num_tokens_from_string
+
 
 class Docx(DocxParser):
     def __init__(self):
@@ -149,7 +151,19 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
                     if not l:
                         break
                     txt += l
-        sections = txt.split("\n")
+        sections = []
+        for sec in txt.split("\n"):
+            if num_tokens_from_string(sec) > 10 * parser_config.get("chunk_token_num", 128):
+                sections.append((sec[:int(len(sec)/2)], ""))
+                sections.append((sec[int(len(sec)/2):], ""))
+            else:
+                sections.append((sec, ""))
+
+        callback(0.8, "Finish parsing.")
+
+    elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE):
+        callback(0.1, "Start to parse.")
+        sections = HtmlParser()(filename, binary)
         sections = [(l, "") for l in sections if l]
         callback(0.8, "Finish parsing.")
 
@@ -163,7 +177,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
 
     else:
         raise NotImplementedError(
-            "file type not supported yet(doc, docx, pdf, txt supported)")
+            "file type not supported yet(pdf, xlsx, doc, docx, txt supported)")
 
     st = timer()
     chunks = naive_merge(

rag/app/one.py

@@ -15,7 +15,7 @@ from io import BytesIO
 import re
 from rag.app import laws
 from rag.nlp import rag_tokenizer, tokenize, find_codec
-from deepdoc.parser import PdfParser, ExcelParser, PlainParser
+from deepdoc.parser import PdfParser, ExcelParser, PlainParser, HtmlParser
 
 
 class Pdf(PdfParser):
@@ -97,6 +97,12 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         sections = [s for s in sections if s]
         callback(0.8, "Finish parsing.")
 
+    elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE):
+        callback(0.1, "Start to parse.")
+        sections = HtmlParser()(filename, binary)
+        sections = [s for s in sections if s]
+        callback(0.8, "Finish parsing.")
+
     elif re.search(r"\.doc$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         binary = BytesIO(binary)

rag/llm/__init__.py

@@ -16,17 +16,20 @@
 from .embedding_model import *
 from .chat_model import *
 from .cv_model import *
+from .rerank_model import *
 
 
 EmbeddingModel = {
     "Ollama": OllamaEmbed,
     "OpenAI": OpenAIEmbed,
     "Xinference": XinferenceEmbed,
-    "Tongyi-Qianwen": DefaultEmbedding, #QWenEmbed,
+    "Tongyi-Qianwen": DefaultEmbedding,#QWenEmbed,
     "ZHIPU-AI": ZhipuEmbed,
     "FastEmbed": FastEmbed,
     "Youdao": YoudaoEmbed,
-    "DeepSeek": DefaultEmbedding
+    "BaiChuan": BaiChuanEmbed,
+    "Jina": JinaEmbed,
+    "BAAI": DefaultEmbedding
 }
 
 
@@ -47,6 +50,13 @@ ChatModel = {
     "Ollama": OllamaChat,
     "Xinference": XinferenceChat,
     "Moonshot": MoonshotChat,
-    "DeepSeek": DeepSeekChat
+    "DeepSeek": DeepSeekChat,
+    "BaiChuan": BaiChuanChat
 }
 
+
+RerankModel = {
+    "BAAI": DefaultRerank,
+    "Jina": JinaRerank,
+    "Youdao": YoudaoRerank,
+}

rag/llm/chat_model.py

@@ -19,6 +19,7 @@ from abc import ABC
 from openai import OpenAI
 import openai
 from ollama import Client
+from volcengine.maas.v2 import MaasService
 from rag.nlp import is_english
 from rag.utils import num_tokens_from_string
 
@@ -56,7 +57,7 @@ class Base(ABC):
                 stream=True,
                 **gen_conf)
             for resp in response:
-                if not resp.choices[0].delta.content:continue
+                if not resp.choices or not resp.choices[0].delta.content:continue
                 ans += resp.choices[0].delta.content
                 total_tokens += 1
                 if resp.choices[0].finish_reason == "length":
@@ -94,6 +95,84 @@ class DeepSeekChat(Base):
         super().__init__(key, model_name, base_url)
 
 
+class BaiChuanChat(Base):
+    def __init__(self, key, model_name="Baichuan3-Turbo", base_url="https://api.baichuan-ai.com/v1"):
+        if not base_url:
+            base_url = "https://api.baichuan-ai.com/v1"
+        super().__init__(key, model_name, base_url)
+
+    @staticmethod
+    def _format_params(params):
+        return {
+            "temperature": params.get("temperature", 0.3),
+            "max_tokens": params.get("max_tokens", 2048),
+            "top_p": params.get("top_p", 0.85),
+        }
+
+    def chat(self, system, history, gen_conf):
+        if system:
+            history.insert(0, {"role": "system", "content": system})
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=history,
+                extra_body={
+                    "tools": [{
+                        "type": "web_search",
+                        "web_search": {
+                            "enable": True,
+                            "search_mode": "performance_first"
+                        }
+                    }]
+                },
+                **self._format_params(gen_conf))
+            ans = response.choices[0].message.content.strip()
+            if response.choices[0].finish_reason == "length":
+                ans += "...\nFor the content length reason, it stopped, continue?" if is_english(
+                    [ans]) else "······\n由于长度的原因，回答被截断了，要继续吗？"
+            return ans, response.usage.total_tokens
+        except openai.APIError as e:
+            return "**ERROR**: " + str(e), 0
+
+    def chat_streamly(self, system, history, gen_conf):
+        if system:
+            history.insert(0, {"role": "system", "content": system})
+        ans = ""
+        total_tokens = 0
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=history,
+                extra_body={
+                    "tools": [{
+                        "type": "web_search",
+                        "web_search": {
+                            "enable": True,
+                            "search_mode": "performance_first"
+                        }
+                    }]
+                },
+                stream=True,
+                **self._format_params(gen_conf))
+            for resp in response:
+                if resp.choices[0].finish_reason == "stop":
+                    if not resp.choices[0].delta.content:
+                        continue
+                    total_tokens = resp.usage.get('total_tokens', 0)
+                if not resp.choices[0].delta.content:
+                    continue
+                ans += resp.choices[0].delta.content
+                if resp.choices[0].finish_reason == "length":
+                    ans += "...\nFor the content length reason, it stopped, continue?" if is_english(
+                        [ans]) else "······\n由于长度的原因，回答被截断了，要继续吗？"
+                yield ans
+
+        except Exception as e:
+            yield ans + "\n**ERROR**: " + str(e)
+
+        yield total_tokens
+
+
 class QWenChat(Base):
     def __init__(self, key, model_name=Generation.Models.qwen_turbo, **kwargs):
         import dashscope
@@ -127,6 +206,7 @@ class QWenChat(Base):
         if system:
             history.insert(0, {"role": "system", "content": system})
         ans = ""
+        tk_count = 0
         try:
             response = Generation.call(
                 self.model_name,
@@ -135,7 +215,6 @@ class QWenChat(Base):
                 stream=True,
                 **gen_conf
             )
-            tk_count = 0
             for resp in response:
                 if resp.status_code == HTTPStatus.OK:
                     ans = resp.output.choices[0]['message']['content']
@@ -182,6 +261,7 @@ class ZhipuChat(Base):
         if "presence_penalty" in gen_conf: del gen_conf["presence_penalty"]
         if "frequency_penalty" in gen_conf: del gen_conf["frequency_penalty"]
         ans = ""
+        tk_count = 0
         try:
             response = self.client.chat.completions.create(
                 model=self.model_name,
@@ -189,7 +269,6 @@ class ZhipuChat(Base):
                 stream=True,
                 **gen_conf
             )
-            tk_count = 0
             for resp in response:
                 if not resp.choices[0].delta.content:continue
                 delta = resp.choices[0].delta.content
@@ -224,7 +303,8 @@ class OllamaChat(Base):
             response = self.client.chat(
                 model=self.model_name,
                 messages=history,
-                options=options
+                options=options,
+                keep_alive=-1
             )
             ans = response["message"]["content"].strip()
             return ans, response["eval_count"] + response.get("prompt_eval_count", 0)
@@ -246,7 +326,8 @@ class OllamaChat(Base):
                 model=self.model_name,
                 messages=history,
                 stream=True,
-                options=options
+                options=options,
+                keep_alive=-1
             )
             for resp in response:
                 if resp["done"]:
@@ -314,3 +395,72 @@ class LocalLLM(Base):
             yield answer + "\n**ERROR**: " + str(e)
 
         yield token_count
+
+
+class VolcEngineChat(Base):
+    def __init__(self, key, model_name, base_url):
+        """
+        Since do not want to modify the original database fields, and the VolcEngine authentication method is quite special,
+        Assemble ak, sk, ep_id into api_key, store it as a dictionary type, and parse it for use
+        model_name is for display only
+        """
+        self.client = MaasService('maas-api.ml-platform-cn-beijing.volces.com', 'cn-beijing')
+        self.volc_ak = eval(key).get('volc_ak', '')
+        self.volc_sk = eval(key).get('volc_sk', '')
+        self.client.set_ak(self.volc_ak)
+        self.client.set_sk(self.volc_sk)
+        self.model_name = eval(key).get('ep_id', '')
+
+    def chat(self, system, history, gen_conf):
+        if system:
+            history.insert(0, {"role": "system", "content": system})
+        try:
+            req = {
+                "parameters": {
+                    "min_new_tokens": gen_conf.get("min_new_tokens", 1),
+                    "top_k": gen_conf.get("top_k", 0),
+                    "max_prompt_tokens": gen_conf.get("max_prompt_tokens", 30000),
+                    "temperature": gen_conf.get("temperature", 0.1),
+                    "max_new_tokens": gen_conf.get("max_tokens", 1000),
+                    "top_p": gen_conf.get("top_p", 0.3),
+                },
+                "messages": history
+            }
+            response = self.client.chat(self.model_name, req)
+            ans = response.choices[0].message.content.strip()
+            if response.choices[0].finish_reason == "length":
+                ans += "...\nFor the content length reason, it stopped, continue?" if is_english(
+                    [ans]) else "······\n由于长度的原因，回答被截断了，要继续吗？"
+            return ans, response.usage.total_tokens
+        except Exception as e:
+            return "**ERROR**: " + str(e), 0
+
+    def chat_streamly(self, system, history, gen_conf):
+        if system:
+            history.insert(0, {"role": "system", "content": system})
+        ans = ""
+        tk_count = 0
+        try:
+            req = {
+                "parameters": {
+                    "min_new_tokens": gen_conf.get("min_new_tokens", 1),
+                    "top_k": gen_conf.get("top_k", 0),
+                    "max_prompt_tokens": gen_conf.get("max_prompt_tokens", 30000),
+                    "temperature": gen_conf.get("temperature", 0.1),
+                    "max_new_tokens": gen_conf.get("max_tokens", 1000),
+                    "top_p": gen_conf.get("top_p", 0.3),
+                },
+                "messages": history
+            }
+            stream = self.client.stream_chat(self.model_name, req)
+            for resp in stream:
+                if not resp.choices[0].message.content:
+                    continue
+                ans += resp.choices[0].message.content
+                if resp.choices[0].finish_reason == "stop":
+                    tk_count = resp.usage.total_tokens
+                yield ans
+
+        except Exception as e:
+            yield ans + "\n**ERROR**: " + str(e)
+        yield tk_count

rag/llm/embedding_model.py

@@ -13,8 +13,10 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import re
 from typing import Optional
 
+import requests
 from huggingface_hub import snapshot_download
 from zhipuai import ZhipuAI
 import os
@@ -26,21 +28,9 @@ from FlagEmbedding import FlagModel
 import torch
 import numpy as np
 
-from api.utils.file_utils import get_project_base_directory, get_home_cache_dir
+from api.utils.file_utils import get_home_cache_dir
 from rag.utils import num_tokens_from_string, truncate
 
-try:
-    flag_model = FlagModel(os.path.join(get_home_cache_dir(), "bge-large-zh-v1.5"),
-        query_instruction_for_retrieval="为这个句子生成表示以用于检索相关文章：",
-        use_fp16=torch.cuda.is_available())
-except Exception as e:
-    model_dir = snapshot_download(repo_id="BAAI/bge-large-zh-v1.5",
-                                  local_dir=os.path.join(get_home_cache_dir(), "bge-large-zh-v1.5"),
-                                  local_dir_use_symlinks=False)
-    flag_model = FlagModel(model_dir,
-                           query_instruction_for_retrieval="为这个句子生成表示以用于检索相关文章：",
-                           use_fp16=torch.cuda.is_available())
-
 
 class Base(ABC):
     def __init__(self, key, model_name):
@@ -54,7 +44,9 @@ class Base(ABC):
 
 
 class DefaultEmbedding(Base):
-    def __init__(self, *args, **kwargs):
+    _model = None
+
+    def __init__(self, key, model_name, **kwargs):
         """
         If you have trouble downloading HuggingFace models, -_^ this might help!!
 
@@ -66,7 +58,18 @@ class DefaultEmbedding(Base):
         ^_-
 
         """
-        self.model = flag_model
+        if not DefaultEmbedding._model:
+            try:
+                self._model = FlagModel(os.path.join(get_home_cache_dir(), re.sub(r"^[a-zA-Z]+/", "", model_name)),
+                                        query_instruction_for_retrieval="为这个句子生成表示以用于检索相关文章：",
+                                        use_fp16=torch.cuda.is_available())
+            except Exception as e:
+                model_dir = snapshot_download(repo_id="BAAI/bge-large-zh-v1.5",
+                                                local_dir=os.path.join(get_home_cache_dir(), re.sub(r"^[a-zA-Z]+/", "", model_name)),
+                                                local_dir_use_symlinks=False)
+                self._model = FlagModel(model_dir,
+                                        query_instruction_for_retrieval="为这个句子生成表示以用于检索相关文章：",
+                                        use_fp16=torch.cuda.is_available())
 
     def encode(self, texts: list, batch_size=32):
         texts = [truncate(t, 2048) for t in texts]
@@ -75,12 +78,12 @@ class DefaultEmbedding(Base):
             token_count += num_tokens_from_string(t)
         res = []
         for i in range(0, len(texts), batch_size):
-            res.extend(self.model.encode(texts[i:i + batch_size]).tolist())
+            res.extend(self._model.encode(texts[i:i + batch_size]).tolist())
         return np.array(res), token_count
 
     def encode_queries(self, text: str):
         token_count = num_tokens_from_string(text)
-        return self.model.encode_queries([text]).tolist()[0], token_count
+        return self._model.encode_queries([text]).tolist()[0], token_count
 
 
 class OpenAIEmbed(Base):
@@ -104,6 +107,15 @@ class OpenAIEmbed(Base):
         return np.array(res.data[0].embedding), res.usage.total_tokens
 
 
+class BaiChuanEmbed(OpenAIEmbed):
+    def __init__(self, key,
+                 model_name='Baichuan-Text-Embedding',
+                 base_url='https://api.baichuan-ai.com/v1'):
+        if not base_url:
+            base_url = "https://api.baichuan-ai.com/v1"
+        super().__init__(key, model_name, base_url)
+
+
 class QWenEmbed(Base):
     def __init__(self, key, model_name="text_embedding_v2", **kwargs):
         dashscope.api_key = key
@@ -180,16 +192,19 @@ class OllamaEmbed(Base):
 
 
 class FastEmbed(Base):
+    _model = None
+
     def __init__(
-        self,
-        key: Optional[str] = None,
-        model_name: str = "BAAI/bge-small-en-v1.5",
-        cache_dir: Optional[str] = None,
-        threads: Optional[int] = None,
-        **kwargs,
+            self,
+            key: Optional[str] = None,
+            model_name: str = "BAAI/bge-small-en-v1.5",
+            cache_dir: Optional[str] = None,
+            threads: Optional[int] = None,
+            **kwargs,
     ):
         from fastembed import TextEmbedding
-        self._model = TextEmbedding(model_name, cache_dir, threads, **kwargs)
+        if not FastEmbed._model:
+            self._model = TextEmbedding(model_name, cache_dir, threads, **kwargs)
 
     def encode(self, texts: list, batch_size=32):
         # Using the internal tokenizer to encode the texts and get the total
@@ -256,3 +271,29 @@ class YoudaoEmbed(Base):
     def encode_queries(self, text):
         embds = YoudaoEmbed._client.encode([text])
         return np.array(embds[0]), num_tokens_from_string(text)
+
+
+class JinaEmbed(Base):
+    def __init__(self, key, model_name="jina-embeddings-v2-base-zh",
+                 base_url="https://api.jina.ai/v1/embeddings"):
+
+        self.base_url = "https://api.jina.ai/v1/embeddings"
+        self.headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {key}"
+        }
+        self.model_name = model_name
+
+    def encode(self, texts: list, batch_size=None):
+        texts = [truncate(t, 8196) for t in texts]
+        data = {
+            "model": self.model_name,
+            "input": texts,
+            'encoding_type': 'float'
+        }
+        res = requests.post(self.base_url, headers=self.headers, json=data).json()
+        return np.array([d["embedding"] for d in res["data"]]), res["usage"]["total_tokens"]
+
+    def encode_queries(self, text):
+        embds, cnt = self.encode([text])
+        return np.array(embds[0]), cnt

rag/llm/rerank_model.py

@@ -0,0 +1,116 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import re
+import requests
+import torch
+from FlagEmbedding import FlagReranker
+from huggingface_hub import snapshot_download
+import os
+from abc import ABC
+import numpy as np
+from api.utils.file_utils import get_home_cache_dir
+from rag.utils import num_tokens_from_string, truncate
+
+def sigmoid(x):
+    return 1 / (1 + np.exp(-x))
+
+class Base(ABC):
+    def __init__(self, key, model_name):
+        pass
+
+    def similarity(self, query: str, texts: list):
+        raise NotImplementedError("Please implement encode method!")
+
+
+class DefaultRerank(Base):
+    _model = None
+
+    def __init__(self, key, model_name, **kwargs):
+        """
+        If you have trouble downloading HuggingFace models, -_^ this might help!!
+
+        For Linux:
+        export HF_ENDPOINT=https://hf-mirror.com
+
+        For Windows:
+        Good luck
+        ^_-
+
+        """
+        if not DefaultRerank._model:
+            try:
+                self._model = FlagReranker(os.path.join(get_home_cache_dir(), re.sub(r"^[a-zA-Z]+/", "", model_name)),
+                                           use_fp16=torch.cuda.is_available())
+            except Exception as e:
+                self._model = snapshot_download(repo_id=model_name,
+                                                local_dir=os.path.join(get_home_cache_dir(),
+                                                                       re.sub(r"^[a-zA-Z]+/", "", model_name)),
+                                                local_dir_use_symlinks=False)
+                self._model = FlagReranker(os.path.join(get_home_cache_dir(), model_name),
+                                           use_fp16=torch.cuda.is_available())
+
+    def similarity(self, query: str, texts: list):
+        pairs = [(query,truncate(t, 2048)) for t in texts]
+        token_count = 0
+        for _, t in pairs:
+            token_count += num_tokens_from_string(t)
+        batch_size = 32
+        res = []
+        for i in range(0, len(pairs), batch_size):
+            scores = self._model.compute_score(pairs[i:i + batch_size], max_length=2048)
+            scores = sigmoid(np.array(scores))
+            res.extend(scores)
+        return np.array(res), token_count
+
+
+class JinaRerank(Base):
+    def __init__(self, key, model_name="jina-reranker-v1-base-en",
+                 base_url="https://api.jina.ai/v1/rerank"):
+        self.base_url = "https://api.jina.ai/v1/rerank"
+        self.headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {key}"
+        }
+        self.model_name = model_name
+
+    def similarity(self, query: str, texts: list):
+        texts = [truncate(t, 8196) for t in texts]
+        data = {
+            "model": self.model_name,
+            "query": query,
+            "documents": texts,
+            "top_n": len(texts)
+        }
+        res = requests.post(self.base_url, headers=self.headers, json=data).json()
+        return np.array([d["relevance_score"] for d in res["results"]]), res["usage"]["total_tokens"]
+
+
+class YoudaoRerank(DefaultRerank):
+    _model = None
+
+    def __init__(self, key=None, model_name="maidalun1020/bce-reranker-base_v1", **kwargs):
+        from BCEmbedding import RerankerModel
+        if not YoudaoRerank._model:
+            try:
+                print("LOADING BCE...")
+                YoudaoRerank._model = RerankerModel(model_name_or_path=os.path.join(
+                    get_home_cache_dir(),
+                    re.sub(r"^[a-zA-Z]+/", "", model_name)))
+            except Exception as e:
+                YoudaoRerank._model = RerankerModel(
+                    model_name_or_path=model_name.replace(
+                        "maidalun1020", "InfiniFlow"))
+

rag/nlp/query.py

@@ -44,7 +44,7 @@ class EsQueryer:
 
     def question(self, txt, tbl="qa", min_match="60%"):
         txt = re.sub(
-            r"[ \r\n\t,，。？?/`!！&\^%%]+",
+            r"[ :\r\n\t,，。？?/`!！&\^%%]+",
             " ",
             rag_tokenizer.tradi2simp(
                 rag_tokenizer.strQ2B(
@@ -54,7 +54,8 @@ class EsQueryer:
         if not self.isChinese(txt):
             tks = rag_tokenizer.tokenize(txt).split(" ")
             tks_w = self.tw.weights(tks)
-            q = [re.sub(r"[ \\\"']+", "", tk)+"^{:.4f}".format(w) for tk, w in tks_w]
+            tks_w = [(re.sub(r"[ \\\"']+", "", tk), w) for tk, w in tks_w]
+            q = ["{}^{:.4f}".format(tk, w) for tk, w in tks_w if tk]
             for i in range(1, len(tks_w)):
                 q.append("\"%s %s\"^%.4f" % (tks_w[i - 1][0], tks_w[i][0], max(tks_w[i - 1][1], tks_w[i][1])*2))
             if not q:
@@ -136,7 +137,11 @@ class EsQueryer:
         from sklearn.metrics.pairwise import cosine_similarity as CosineSimilarity
         import numpy as np
         sims = CosineSimilarity([avec], bvecs)
+        tksim = self.token_similarity(atks, btkss)
+        return np.array(sims[0]) * vtweight + \
+            np.array(tksim) * tkweight, tksim, sims[0]
 
+    def token_similarity(self, atks, btkss):
         def toDict(tks):
             d = {}
             if isinstance(tks, str):
@@ -149,9 +154,7 @@ class EsQueryer:
 
         atks = toDict(atks)
         btkss = [toDict(tks) for tks in btkss]
-        tksim = [self.similarity(atks, btks) for btks in btkss]
-        return np.array(sims[0]) * vtweight + \
-            np.array(tksim) * tkweight, tksim, sims[0]
+        return [self.similarity(atks, btks) for btks in btkss]
 
     def similarity(self, qtwt, dtwt):
         if isinstance(dtwt, type("")):

rag/nlp/rag_tokenizer.py

@@ -24,7 +24,7 @@ class RagTokenizer:
     def loadDict_(self, fnm):
         print("[HUQIE]:Build trie", fnm, file=sys.stderr)
         try:
-            of = open(fnm, "r")
+            of = open(fnm, "r", encoding='utf-8')
             while True:
                 line = of.readline()
                 if not line:
@@ -241,11 +241,14 @@ class RagTokenizer:
 
         return self.score_(res[::-1])
 
+    def english_normalize_(self, tks):
+        return [self.stemmer.stem(self.lemmatizer.lemmatize(t)) if re.match(r"[a-zA-Z_-]+$", t) else t for t in tks]
+
     def tokenize(self, line):
         line = self._strQ2B(line).lower()
         line = self._tradi2simp(line)
         zh_num = len([1 for c in line if is_chinese(c)])
-        if zh_num < len(line) * 0.2:
+        if zh_num == 0:
             return " ".join([self.stemmer.stem(self.lemmatizer.lemmatize(t)) for t in word_tokenize(line)])
 
         arr = re.split(self.SPLIT_CHAR, line)
@@ -293,7 +296,7 @@ class RagTokenizer:
 
                 i = e + 1
 
-        res = " ".join(res)
+        res = " ".join(self.english_normalize_(res))
         if self.DEBUG:
             print("[TKS]", self.merge_(res))
         return self.merge_(res)
@@ -336,7 +339,7 @@ class RagTokenizer:
 
             res.append(stk)
 
-        return " ".join(res)
+        return " ".join(self.english_normalize_(res))
 
 
 def is_chinese(s):

rag/nlp/search.py

@@ -71,8 +71,8 @@ class Dealer:
 
         s = Search()
         pg = int(req.get("page", 1)) - 1
-        ps = int(req.get("size", 1000))
         topk = int(req.get("topk", 1024))
+        ps = int(req.get("size", topk))
         src = req.get("fields", ["docnm_kwd", "content_ltks", "kb_id", "img_id", "title_tks", "important_kwd",
                                  "image_id", "doc_id", "q_512_vec", "q_768_vec", "position_int",
                                  "q_1024_vec", "q_1536_vec", "available_int", "content_with_weight"])
@@ -311,6 +311,26 @@ class Dealer:
                                                         ins_tw, tkweight, vtweight)
         return sim, tksim, vtsim
 
+    def rerank_by_model(self, rerank_mdl, sres, query, tkweight=0.3,
+               vtweight=0.7, cfield="content_ltks"):
+        _, keywords = self.qryr.question(query)
+
+        for i in sres.ids:
+            if isinstance(sres.field[i].get("important_kwd", []), str):
+                sres.field[i]["important_kwd"] = [sres.field[i]["important_kwd"]]
+        ins_tw = []
+        for i in sres.ids:
+            content_ltks = sres.field[i][cfield].split(" ")
+            title_tks = [t for t in sres.field[i].get("title_tks", "").split(" ") if t]
+            important_kwd = sres.field[i].get("important_kwd", [])
+            tks = content_ltks + title_tks + important_kwd
+            ins_tw.append(tks)
+
+        tksim = self.qryr.token_similarity(keywords, ins_tw)
+        vtsim,_ = rerank_mdl.similarity(" ".join(keywords), [rmSpace(" ".join(tks)) for tks in ins_tw])
+
+        return tkweight*np.array(tksim) + vtweight*vtsim, tksim, vtsim
+
     def hybrid_similarity(self, ans_embd, ins_embd, ans, inst):
         return self.qryr.hybrid_similarity(ans_embd,
                                            ins_embd,
@@ -318,17 +338,22 @@ class Dealer:
                                            rag_tokenizer.tokenize(inst).split(" "))
 
     def retrieval(self, question, embd_mdl, tenant_id, kb_ids, page, page_size, similarity_threshold=0.2,
-                  vector_similarity_weight=0.3, top=1024, doc_ids=None, aggs=True):
+                  vector_similarity_weight=0.3, top=1024, doc_ids=None, aggs=True, rerank_mdl=None):
         ranks = {"total": 0, "chunks": [], "doc_aggs": {}}
         if not question:
             return ranks
         req = {"kb_ids": kb_ids, "doc_ids": doc_ids, "size": page_size,
                "question": question, "vector": True, "topk": top,
-               "similarity": similarity_threshold}
+               "similarity": similarity_threshold,
+               "available_int": 1}
         sres = self.search(req, index_name(tenant_id), embd_mdl)
 
-        sim, tsim, vsim = self.rerank(
-            sres, question, 1 - vector_similarity_weight, vector_similarity_weight)
+        if rerank_mdl:
+            sim, tsim, vsim = self.rerank_by_model(rerank_mdl,
+                sres, question, 1 - vector_similarity_weight, vector_similarity_weight)
+        else:
+            sim, tsim, vsim = self.rerank(
+                sres, question, 1 - vector_similarity_weight, vector_similarity_weight)
         idx = np.argsort(sim * -1)
 
         dim = len(sres.query_vector)
@@ -407,3 +432,13 @@ class Dealer:
         except Exception as e:
             chat_logger.error(f"SQL failure: {sql} =>" + str(e))
             return {"error": str(e)}
+
+    def chunk_list(self, doc_id, tenant_id, max_count=1024, fields=["docnm_kwd", "content_with_weight", "img_id"]):
+        s = Search()
+        s = s.query(Q("match", doc_id=doc_id))[0:max_count]
+        s = s.to_dict()
+        es_res = self.es.search(s, idxnm=index_name(tenant_id), timeout="600s", src=fields)
+        res = []
+        for index, chunk in enumerate(es_res['hits']['hits']):
+            res.append({fld: chunk['_source'].get(fld) for fld in fields})
+        return res

rag/nlp/term_weight.py

@@ -104,7 +104,7 @@ class Dealer:
         while i < len(tks):
             j = i
             if i == 0 and oneTerm(tks[i]) and len(
-                    tks) > 1 and len(tks[i + 1]) > 1:  # 多 工位
+                    tks) > 1 and (len(tks[i + 1]) > 1 and not re.match(r"[0-9a-zA-Z]", tks[i + 1])):  # 多 工位
                 res.append(" ".join(tks[0:2]))
                 i = 2
                 continue

rag/raptor.py

@@ -0,0 +1,115 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import re
+import traceback
+from concurrent.futures import ThreadPoolExecutor, ALL_COMPLETED, wait
+from threading import Lock
+from typing import Tuple
+import umap
+import numpy as np
+from sklearn.mixture import GaussianMixture
+
+from rag.utils import num_tokens_from_string, truncate
+
+
+class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:
+    def __init__(self, max_cluster, llm_model, embd_model, prompt, max_token=256, threshold=0.1):
+        self._max_cluster = max_cluster
+        self._llm_model = llm_model
+        self._embd_model = embd_model
+        self._threshold = threshold
+        self._prompt = prompt
+        self._max_token = max_token
+
+    def _get_optimal_clusters(self, embeddings: np.ndarray, random_state:int):
+        max_clusters = min(self._max_cluster, len(embeddings))
+        n_clusters = np.arange(1, max_clusters)
+        bics = []
+        for n in n_clusters:
+            gm = GaussianMixture(n_components=n, random_state=random_state)
+            gm.fit(embeddings)
+            bics.append(gm.bic(embeddings))
+        optimal_clusters = n_clusters[np.argmin(bics)]
+        return optimal_clusters
+
+    def __call__(self, chunks: Tuple[str, np.ndarray], random_state, callback=None):
+        layers = [(0, len(chunks))]
+        start, end = 0, len(chunks)
+        if len(chunks) <= 1: return
+
+        def summarize(ck_idx, lock):
+            nonlocal chunks
+            try:
+                texts = [chunks[i][0] for i in ck_idx]
+                len_per_chunk = int((self._llm_model.max_length - self._max_token)/len(texts))
+                cluster_content = "\n".join([truncate(t, max(1, len_per_chunk)) for t in texts])
+                cnt = self._llm_model.chat("You're a helpful assistant.",
+                                             [{"role": "user", "content": self._prompt.format(cluster_content=cluster_content)}],
+                                             {"temperature": 0.3, "max_tokens": self._max_token}
+                                             )
+                cnt = re.sub("(······\n由于长度的原因，回答被截断了，要继续吗？|For the content length reason, it stopped, continue?)", "", cnt)
+                print("SUM:", cnt)
+                embds, _ = self._embd_model.encode([cnt])
+                with lock:
+                    chunks.append((cnt, embds[0]))
+            except Exception as e:
+                print(e, flush=True)
+                traceback.print_stack(e)
+                return e
+
+        labels = []
+        while end - start > 1:
+            embeddings = [embd for _, embd in chunks[start: end]]
+            if len(embeddings) == 2:
+                summarize([start, start+1], Lock())
+                if callback:
+                    callback(msg="Cluster one layer: {} -> {}".format(end-start, len(chunks)-end))
+                labels.extend([0,0])
+                layers.append((end, len(chunks)))
+                start = end
+                end = len(chunks)
+                continue
+
+            n_neighbors = int((len(embeddings) - 1) ** 0.8)
+            reduced_embeddings = umap.UMAP(
+                n_neighbors=max(2, n_neighbors), n_components=min(12, len(embeddings)-2), metric="cosine"
+            ).fit_transform(embeddings)
+            n_clusters = self._get_optimal_clusters(reduced_embeddings, random_state)
+            if n_clusters == 1:
+                lbls = [0 for _ in range(len(reduced_embeddings))]
+            else:
+                gm = GaussianMixture(n_components=n_clusters, random_state=random_state)
+                gm.fit(reduced_embeddings)
+                probs = gm.predict_proba(reduced_embeddings)
+                lbls = [np.where(prob > self._threshold)[0] for prob in probs]
+                lbls = [lbl[0] if isinstance(lbl, np.ndarray) else lbl for lbl in lbls]
+            lock = Lock()
+            with ThreadPoolExecutor(max_workers=12) as executor:
+                threads = []
+                for c in range(n_clusters):
+                    ck_idx = [i+start for i in range(len(lbls)) if lbls[i] == c]
+                    threads.append(executor.submit(summarize, ck_idx, lock))
+                wait(threads, return_when=ALL_COMPLETED)
+                print([t.result() for t in threads])
+
+            assert len(chunks) - end == n_clusters, "{} vs. {}".format(len(chunks) - end, n_clusters)
+            labels.extend(lbls)
+            layers.append((end, len(chunks)))
+            if callback:
+                callback(msg="Cluster one layer: {} -> {}".format(end-start, len(chunks)-end))
+            start = end
+            end = len(chunks)
+

rag/svr/task_executor.py

@@ -26,20 +26,22 @@ import traceback
 from functools import partial
 
 from api.db.services.file2document_service import File2DocumentService
+from api.settings import retrievaler
+from rag.raptor import RecursiveAbstractiveProcessing4TreeOrganizedRetrieval as Raptor
 from rag.utils.minio_conn import MINIO
 from api.db.db_models import close_connection
 from rag.settings import database_logger, SVR_QUEUE_NAME
 from rag.settings import cron_logger, DOC_MAXIMUM_SIZE
 from multiprocessing import Pool
 import numpy as np
-from elasticsearch_dsl import Q
+from elasticsearch_dsl import Q, Search
 from multiprocessing.context import TimeoutError
 from api.db.services.task_service import TaskService
 from rag.utils.es_conn import ELASTICSEARCH
 from timeit import default_timer as timer
-from rag.utils import rmSpace, findMaxTm
+from rag.utils import rmSpace, findMaxTm, num_tokens_from_string
 
-from rag.nlp import search
+from rag.nlp import search, rag_tokenizer
 from io import BytesIO
 import pandas as pd
 
@@ -114,6 +116,8 @@ def collect():
     tasks = TaskService.get_tasks(msg["id"])
     assert tasks, "{} empty task!".format(msg["id"])
     tasks = pd.DataFrame(tasks)
+    if msg.get("type", "") == "raptor":
+        tasks["task_type"] = "raptor"
     return tasks
 
 
@@ -245,6 +249,47 @@ def embedding(docs, mdl, parser_config={}, callback=None):
     return tk_count
 
 
+def run_raptor(row, chat_mdl, embd_mdl, callback=None):
+    vts, _ = embd_mdl.encode(["ok"])
+    vctr_nm = "q_%d_vec"%len(vts[0])
+    chunks = []
+    for d in retrievaler.chunk_list(row["doc_id"], row["tenant_id"], fields=["content_with_weight", vctr_nm]):
+        chunks.append((d["content_with_weight"], np.array(d[vctr_nm])))
+
+    raptor = Raptor(
+        row["parser_config"]["raptor"].get("max_cluster", 64),
+        chat_mdl,
+        embd_mdl,
+        row["parser_config"]["raptor"]["prompt"],
+        row["parser_config"]["raptor"]["max_token"],
+        row["parser_config"]["raptor"]["threshold"]
+    )
+    original_length = len(chunks)
+    raptor(chunks, row["parser_config"]["raptor"]["random_seed"], callback)
+    doc = {
+        "doc_id": row["doc_id"],
+        "kb_id": [str(row["kb_id"])],
+        "docnm_kwd": row["name"],
+        "title_tks": rag_tokenizer.tokenize(row["name"])
+    }
+    res = []
+    tk_count = 0
+    for content, vctr in chunks[original_length:]:
+        d = copy.deepcopy(doc)
+        md5 = hashlib.md5()
+        md5.update((content + str(d["doc_id"])).encode("utf-8"))
+        d["_id"] = md5.hexdigest()
+        d["create_time"] = str(datetime.datetime.now()).replace("T", " ")[:19]
+        d["create_timestamp_flt"] = datetime.datetime.now().timestamp()
+        d[vctr_nm] = vctr.tolist()
+        d["content_with_weight"] = content
+        d["content_ltks"] = rag_tokenizer.tokenize(content)
+        d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
+        res.append(d)
+        tk_count += num_tokens_from_string(content)
+    return res, tk_count
+
+
 def main():
     rows = collect()
     if len(rows) == 0:
@@ -259,35 +304,45 @@ def main():
             cron_logger.error(str(e))
             continue
 
-        st = timer()
-        cks = build(r)
-        cron_logger.info("Build chunks({}): {}".format(r["name"], timer() - st))
-        if cks is None:
-            continue
-        if not cks:
-            callback(1., "No chunk! Done!")
-            continue
-        # TODO: exception handler
-        ## set_progress(r["did"], -1, "ERROR: ")
-        callback(
-            msg="Finished slicing files(%d). Start to embedding the content." %
-                len(cks))
-        st = timer()
-        try:
-            tk_count = embedding(cks, embd_mdl, r["parser_config"], callback)
-        except Exception as e:
-            callback(-1, "Embedding error:{}".format(str(e)))
-            cron_logger.error(str(e))
-            tk_count = 0
-        cron_logger.info("Embedding elapsed({}): {:.2f}".format(r["name"], timer() - st))
+        if r.get("task_type", "") == "raptor":
+            try:
+                chat_mdl = LLMBundle(r["tenant_id"], LLMType.CHAT, llm_name=r["llm_id"], lang=r["language"])
+                cks, tk_count = run_raptor(r, chat_mdl, embd_mdl, callback)
+            except Exception as e:
+                callback(-1, msg=str(e))
+                cron_logger.error(str(e))
+                continue
+        else:
+            st = timer()
+            cks = build(r)
+            cron_logger.info("Build chunks({}): {}".format(r["name"], timer() - st))
+            if cks is None:
+                continue
+            if not cks:
+                callback(1., "No chunk! Done!")
+                continue
+            # TODO: exception handler
+            ## set_progress(r["did"], -1, "ERROR: ")
+            callback(
+                msg="Finished slicing files(%d). Start to embedding the content." %
+                    len(cks))
+            st = timer()
+            try:
+                tk_count = embedding(cks, embd_mdl, r["parser_config"], callback)
+            except Exception as e:
+                callback(-1, "Embedding error:{}".format(str(e)))
+                cron_logger.error(str(e))
+                tk_count = 0
+            cron_logger.info("Embedding elapsed({}): {:.2f}".format(r["name"], timer() - st))
+            callback(msg="Finished embedding({:.2f})! Start to build index!".format(timer() - st))
 
-        callback(msg="Finished embedding({:.2f})! Start to build index!".format(timer() - st))
         init_kb(r)
         chunk_count = len(set([c["_id"] for c in cks]))
         st = timer()
         es_r = ""
-        for b in range(0, len(cks), 32):
-            es_r = ELASTICSEARCH.bulk(cks[b:b + 32], search.index_name(r["tenant_id"]))
+        es_bulk_size = 16
+        for b in range(0, len(cks), es_bulk_size):
+            es_r = ELASTICSEARCH.bulk(cks[b:b + es_bulk_size], search.index_name(r["tenant_id"]))
             if b % 128 == 0:
                 callback(prog=0.8 + 0.1 * (b + 1) / len(cks), msg="")
 

rag/utils/redis_conn.py

@@ -97,15 +97,17 @@ class RedisDB:
         return False
 
     def queue_product(self, queue, message, exp=settings.SVR_QUEUE_RETENTION) -> bool:
-        try:
-            payload = {"message": json.dumps(message)}
-            pipeline = self.REDIS.pipeline()
-            pipeline.xadd(queue, payload)
-            pipeline.expire(queue, exp)
-            pipeline.execute()
-            return True
-        except Exception as e:
-            logging.warning("[EXCEPTION]producer" + str(queue) + "||" + str(e))
+        for _ in range(3):
+            try:
+                payload = {"message": json.dumps(message)}
+                pipeline = self.REDIS.pipeline()
+                pipeline.xadd(queue, payload)
+                pipeline.expire(queue, exp)
+                pipeline.execute()
+                return True
+            except Exception as e:
+                print(e)
+                logging.warning("[EXCEPTION]producer" + str(queue) + "||" + str(e))
         return False
 
     def queue_consumer(self, queue_name, group_name, consumer_name, msg_id=b">") -> Payload:
@@ -115,7 +117,7 @@ class RedisDB:
                 self.REDIS.xgroup_create(
                     queue_name,
                     group_name,
-                    id="$",
+                    id="0",
                     mkstream=True
                 )
             args = {

requirements.txt

@@ -134,3 +134,8 @@ yarl==1.9.4
 zhipuai==2.0.1
 BCEmbedding
 loguru==0.7.2
+umap-learn
+fasttext==0.9.2
+volcengine
+readability-lxml==0.8.1
+html_text==0.6.2

requirements_arm.txt

@@ -0,0 +1,142 @@
+accelerate==0.27.2
+aiohttp==3.9.3
+aiosignal==1.3.1
+annotated-types==0.6.0
+anyio==4.3.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+#Aspose.Slides==24.2.0
+attrs==23.2.0
+blinker==1.7.0
+cachelib==0.12.0
+cachetools==5.3.3
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+coloredlogs==15.0.1
+cryptography==42.0.5
+dashscope==1.14.1
+datasets==2.17.1
+datrie==0.8.2
+demjson3==3.0.6
+dill==0.3.8
+distro==1.9.0
+elastic-transport==8.12.0
+elasticsearch==8.12.1
+elasticsearch-dsl==8.12.0
+et-xmlfile==1.1.0
+filelock==3.13.1
+fastembed==0.2.6
+FlagEmbedding==1.2.5
+Flask==3.0.2
+Flask-Cors==4.0.0
+Flask-Login==0.6.3
+Flask-Session==0.6.0
+flatbuffers==23.5.26
+frozenlist==1.4.1
+fsspec==2023.10.0
+h11==0.14.0
+hanziconv==0.3.2
+httpcore==1.0.4
+httpx==0.27.0
+huggingface-hub==0.20.3
+humanfriendly==10.0
+idna==3.6
+install==1.3.5
+itsdangerous==2.1.2
+Jinja2==3.1.3
+joblib==1.3.2
+lxml==5.1.0
+MarkupSafe==2.1.5
+minio==7.2.4
+mpmath==1.3.0
+multidict==6.0.5
+multiprocess==0.70.16
+networkx==3.2.1
+nltk==3.8.1
+numpy==1.26.4
+# nvidia-cublas-cu12==12.1.3.1
+# nvidia-cuda-cupti-cu12==12.1.105
+# nvidia-cuda-nvrtc-cu12==12.1.105
+# nvidia-cuda-runtime-cu12==12.1.105
+# nvidia-cudnn-cu12==8.9.2.26
+# nvidia-cufft-cu12==11.0.2.54
+# nvidia-curand-cu12==10.3.2.106
+# nvidia-cusolver-cu12==11.4.5.107
+# nvidia-cusparse-cu12==12.1.0.106
+# nvidia-nccl-cu12==2.19.3
+# nvidia-nvjitlink-cu12==12.3.101
+# nvidia-nvtx-cu12==12.1.105
+ollama==0.1.9
+# onnxruntime-gpu==1.17.1
+openai==1.12.0
+opencv-python==4.9.0.80
+openpyxl==3.1.2
+packaging==23.2
+pandas==2.2.1
+pdfminer.six==20221105
+pdfplumber==0.10.4
+peewee==3.17.1
+pillow==10.2.0
+protobuf==4.25.3
+psutil==5.9.8
+pyarrow==15.0.0
+pyarrow-hotfix==0.6
+pyclipper==1.3.0.post5
+pycparser==2.21
+pycryptodome==3.20.0
+pycryptodome-test-vectors==1.0.14
+pycryptodomex==3.20.0
+pydantic==2.6.2
+pydantic_core==2.16.3
+PyJWT==2.8.0
+PyMySQL==1.1.0
+PyPDF2==3.0.1
+pypdfium2==4.27.0
+python-dateutil==2.8.2
+python-docx==1.1.0
+python-dotenv==1.0.1
+python-pptx==0.6.23
+pytz==2024.1
+PyYAML==6.0.1
+redis==5.0.3
+regex==2023.12.25
+requests==2.31.0
+ruamel.yaml==0.18.6
+ruamel.yaml.clib==0.2.8
+safetensors==0.4.2
+scikit-learn==1.4.1.post1
+scipy==1.12.0
+sentence-transformers==2.4.0
+shapely==2.0.3
+six==1.16.0
+sniffio==1.3.1
+StrEnum==0.4.15
+sympy==1.12
+threadpoolctl==3.3.0
+tika==2.6.0
+tiktoken==0.6.0
+tokenizers==0.15.2
+torch==2.2.1
+tqdm==4.66.2
+transformers==4.38.1
+# triton==2.2.0
+typing_extensions==4.10.0
+tzdata==2024.1
+urllib3==2.2.1
+Werkzeug==3.0.1
+xgboost==2.0.3
+XlsxWriter==3.2.0
+xpinyin==0.7.6
+xxhash==3.4.1
+yarl==1.9.4
+zhipuai==2.0.1
+BCEmbedding
+loguru==0.7.2
+umap-learn
+fasttext==0.9.2
+volcengine
+opencv-python-headless==4.9.0.80
+readability-lxml==0.8.1
+html_text==0.6.2

requirements_dev.txt

@@ -122,3 +122,8 @@ BCEmbedding
 loguru==0.7.2
 ollama==0.1.8
 redis==5.0.4
+fasttext==0.9.2
+umap-learn
+volcengine
+readability-lxml==0.8.1
+html_text==0.6.2

web/jest-setup.ts

@@ -0,0 +1,2 @@
+import '@testing-library/jest-dom';
+import 'umi/test-setup';

web/jest.config.ts

@@ -0,0 +1,33 @@
+import { Config, configUmiAlias, createConfig } from 'umi/test';
+
+export default async () => {
+  return (await configUmiAlias({
+    ...createConfig({
+      target: 'browser',
+      jsTransformer: 'esbuild',
+      // config opts for esbuild , it will pass to esbuild directly
+      jsTransformerOpts: { jsx: 'automatic' },
+    }),
+    setupFilesAfterEnv: ['<rootDir>/jest-setup.ts'],
+    collectCoverageFrom: [
+      '**/*.{ts,tsx,js,jsx}',
+      '!.umi/**',
+      '!.umi-test/**',
+      '!.umi-production/**',
+      '!.umirc.{js,ts}',
+      '!.umirc.*.{js,ts}',
+      '!jest.config.{js,ts}',
+      '!coverage/**',
+      '!dist/**',
+      '!config/**',
+      '!mock/**',
+    ],
+    // if you require some es-module npm package, please uncomment below line and insert your package name
+    // transformIgnorePatterns: ['node_modules/(?!.*(lodash-es|your-es-pkg-name)/)']
+    coverageThreshold: {
+      global: {
+        lines: 1,
+      },
+    },
+  })) as Config.InitialOptions;
+};

web/package.json

@@ -7,7 +7,8 @@
     "postinstall": "umi setup",
     "lint": "umi lint --eslint-only",
     "setup": "umi setup",
-    "start": "npm run dev"
+    "start": "npm run dev",
+    "test": "jest --no-cache --coverage"
   },
   "dependencies": {
     "@ant-design/icons": "^5.2.6",
@@ -18,7 +19,9 @@
     "antd": "^5.12.7",
     "axios": "^1.6.3",
     "classnames": "^2.5.1",
+    "dagre": "^0.8.5",
     "dayjs": "^1.11.10",
+    "elkjs": "^0.9.3",
     "eventsource-parser": "^1.1.2",
     "i18next": "^23.7.16",
     "i18next-browser-languagedetector": "^8.0.0",
@@ -45,20 +48,28 @@
   },
   "devDependencies": {
     "@react-dev-inspector/umi4-plugin": "^2.0.1",
+    "@testing-library/jest-dom": "^6.4.5",
+    "@testing-library/react": "^15.0.7",
+    "@types/dagre": "^0.7.52",
+    "@types/jest": "^29.5.12",
     "@types/lodash": "^4.14.202",
     "@types/react": "^18.0.33",
     "@types/react-copy-to-clipboard": "^5.0.7",
     "@types/react-dom": "^18.0.11",
     "@types/react-syntax-highlighter": "^15.5.11",
+    "@types/testing-library__jest-dom": "^6.0.0",
     "@types/uuid": "^9.0.8",
     "@types/webpack-env": "^1.18.4",
     "@umijs/lint": "^4.1.1",
     "@umijs/plugins": "^4.1.0",
     "cross-env": "^7.0.3",
+    "jest": "^29.7.0",
+    "jest-environment-jsdom": "^29.7.0",
     "prettier": "^3.2.4",
     "prettier-plugin-organize-imports": "^3.2.4",
     "prettier-plugin-packagejson": "^2.4.9",
     "react-dev-inspector": "^2.0.1",
+    "ts-node": "^10.9.2",
     "typescript": "^5.0.3",
     "umi-plugin-icons": "^0.1.1"
   }

web/src/assets/svg/llm/baichuan.svg

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" width="256px" height="256px" viewBox="0 0 256 256" enable-background="new 0 0 256 256" xml:space="preserve">  <image id="image0" width="256" height="256" x="0" y="0"
+    xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAMAAABrrFhUAAAAIGNIUk0AAHolAACAgwAA+f8AAIDp
+AAB1MAAA6mAAADqYAAAXb5JfxUYAAACuUExURf////349fzy7/77+f349fzy7//+/fXOv/bTxf//
+/+2njfjg1u+xmfG6pumXd/jg1uaDXv349fLArfPIt++xmfvv6u6tlP77+frp4uqbfeaIZPPEsvC2
+n/LArf////LCr/////G6pvLCr++xmfC2n/zy7/fXyu2njeiPbf318fnl3eudfvG+quykif/////+
+/eBnOeN2TON3TuJ0SuFvROFqPeN5UOV+V+R8VP///3/zEGYAAAAwdFJOUwA+RTlbYVqZk1i/Y7yt
+4YH5Yquhv27DXnbb86W1plCpJZiOq7lojcvrZnrZrc8yNOA7KFcAAAABYktHRACIBR1IAAAACXBI
+WXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH6AUbBTg4QG2qLwAAAyhJREFUeNrt3dlS2zAUgGHaUjvq
+ljht4pB0oy3ZoDTgLH3/JytThinjWNJxhwmco/+/RkL6RngSc6GjIyIiIiIiIiIiIiIiIiIiIiIi
+IiIiIiIiIvq/nj0X9qI28PilsOPH2FaWe+q42k++Wgl7XRv4Rjrw7WMAdHyruUoDwL1LHKDrXU0a
+AFkvbQCX+1eTBEBxnThAP7CaFABcL20A9z60mgQAiiptAPchuBr7AIN12gBuGF6NeYCyCq/GOkDk
+CWAfYLRJG8CdxFZjHKAbXY1tgPgBMA4wiD0BjAP4X4QlAtARrMYyQPBrcAoAuWQ1hgHcOHGA+GcA
+2wBlL22A8IuwBACKSrYaswDCA2AWIPDPsCQA3ES6mocD+FgJ+3QAgHJ7eIDP4g4AIH0CPCTAk+oL
+AAAAAAAAAAAAwAEBvoozCnC6FfbNKMB36cBTAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAgP3cDzHAziTA2Vq6mFXfIkCLA3Bd1gdbAMjEtwusJvUDYAFAesHMTbtsb7QB
+gLMrMcB07wAYAHAz8f7n+/s3ACC4ZfGuvGG4eoDYTcv3GmcN49UDLOSfAZZN47UDRG9a/te5a5pA
+O8BS/ARYDxonUA7g5uIDMGw8ANoBZLcs/m3UPINuAHcu3v+s+QAoBxiJ918VnilUAwiu2r7rwnMA
+VAO4qXj/u9I3iWaA7LcYYOI7AKoBJFdt3zYvvJMoBmjxGWDqn0UxQIsD4Pyz6AUQ3bV+WzcwjVoA
+1xfv/2fgAOgFkH8NXhehebQCtDgAs9ABUAtQ7qS/fzMKTqQUwF2ID8AweAC0AkjvWr95AizCM+kE
+aPEqfBI+AEoB5F+De5H96wRo8Sa0H5tLJcBC/ATYFrG5NAK0eALksb8AnQDiV+Hj6P51Akh/9WYZ
+n8w0wEn8AOgEWApbCCa7/CXssjbwSd07TEREREREREREREREREREREREREREevoDtKBqvEP0IEYA
+AAAldEVYdGRhdGU6Y3JlYXRlADIwMjQtMDUtMjdUMDU6NTY6NTYrMDA6MDANJESyAAAAJXRFWHRk
+YXRlOm1vZGlmeQAyMDI0LTA1LTI3VDA1OjU2OjU2KzAwOjAwfHn8DgAAACh0RVh0ZGF0ZTp0aW1l
+c3RhbXAAMjAyNC0wNS0yN1QwNTo1Njo1NiswMDowMCts3dEAAAAASUVORK5CYII=" />
+</svg>

web/src/assets/svg/llm/jina.svg

@@ -0,0 +1,228 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" width="200px" height="200px" viewBox="0 0 200 200" enable-background="new 0 0 200 200" xml:space="preserve">  <image id="image0" width="200" height="200" x="0" y="0"
+    xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAIAAAAiOjnJAAAAIGNIUk0AAHomAACAhAAA+gAAAIDo
+AAB1MAAA6mAAADqYAAAXcJy6UTwAAAAGYktHRAD/AP8A/6C9p5MAAAKjelRYdFJhdyBwcm9maWxl
+IHR5cGUgeG1wAABIiZ1WS5bjMAjcc4o+ggQS2Mdx2/Ku35vlHH8K+RP/k+n4tTq2oShKgEJ/f/7Q
+Fz6RWUl6a6SzRgcLOmi2pJGD32uvxcTfycCmUZOOypqlm56v1iObFAodh/0FaIaTSpc5paQH2Omd
+QzeWcAXpADWS1Q8XgxUXHWFoPEqU1i98C8JYGeswgeC/WAtYsLGGBw/AQk4rdE+8QCqbqOC+tQD3
+FikW8JzfI2lWMEKeTpTtDLZNCU5yTgjAGYA9F1qScUOsGY7ZZXTqoIwkucgs6BXvGqCzRKCb3Pna
+bE0vag/mDzYE0r22Sxr1LmpR8zSqlFBIFbJmh3JeCRbQDCnBS6pmot80PeAB2FF6HlcJAX/N4GWz
+5UlnsyXr56Q3cHVX6bitvy0G2lfDp8WQYi0CgClPRUHbqqg1WnRj5MBzzAJJRx1gha5bCyMshUHn
+JHbRXFC4PRdHxh6SxWejuTjmO0yIiLzhmJNBI2wFEsvsQFKVYRDPd9V74OhCZxcAKzYfHYvGJvT8
+gAqKWAP6Hdv67HYXiDYu2dCQqa1Rgu+ix8IaJSNMq1ESxB2wttI4AbxDHspOgvCoRVl4NMEqrgUK
+8sxsE+aKFR3MMbdmThl8/A9BEMrZRDF8ywgdzoHo5fpe5px8Y8C6WffMpa6g9M7wuc9erX3RtNsy
+u6uyk4KR7uv22N/P7U0vx/8b9sf23qW2O7/C9fm1NPZyXC7NTH4S1NNq6ma5H/MztI8QnwmNHxsY
+vB0YAxiM5lRaNTxA11V1Lp2elKTPpHyvJH0m5a2S9ceHW9JnUiKBBhMgHOfiC55WyhP2dmwge691
+XD1EFT/vDCdc1eVkTXvz3xbmc4vUX0e34IefZwSqhf4BDtczzLoeiJ0AAC2/SURBVHja7b1psCTZ
+dd/3P+fcm5lV9dbununZQaAHIGYGM+AYg4UACYEEIJkiFgUXwIJFiaJAh0iFbckhOvRBXsIOhW2F
+w6IVFm2RoigjHKBtEjAgkACHIAYCsRADYAgCxEZgMPtM72+tJfMu5/hDVtWr18ts3Q/TaOUvMrpf
+V+erzLr3n+eee+45t4COjo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6O
+jo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6O
+jo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6Ojo6LQy/0DVwh8P5/6gt9P9/3dMICwAtH
+K6nUaesSkRf6Br7XVFWlqsxsZu0rRBApzHwhfecKgL1zBq16XjXPzup4bvz7aLFKXwjzuKmJ6Nix
+YzfeeKNwr3CHtrdGjz/+8FMnHlEEoCaGE4kxv9D3+33JVSws3hvOCLC9f1biHeONP/am219xx/U3
+3ri2diQFR7q8tnZkd7j54He/cd8nP/qnX/5CTBNimF78bTsuzlU8FBIwH8YYAGj6z7vvfMUv/p33
+/tCddxLUDDGyqQ+jPN4ae8ONR695zavvvu6aQ1/7xldyzPtHQp698+Kbd1yAq1pYNO97WlpZy8im
++Ydf+5q/9XN/8+jhI8OtTYG5ouJyiaVaRbkMX6pR01Qcjv3A9be+7NY/+8qXY4p770JMRAQhMBER
+gc7jhf7UVwpXr7Bo0aJYiLVpftVrbn/ve3/B1CxmX3gpSXyv9CvLbvX6weFDxWCtWF6tSm22WXT9
+8DVr64c+/8ADAIgBAxGmeiXrBsSn5+oVVgsBgHclWVpdwb/4tX9yw423pAwRP2qa3vWHltfWb+xd
+d0N55FC1vlqurpRLy1VFaHIKRnLzLS/ZGY4fe+KpnGNZ9nJOIJ0Kqx0Pr2Yn9ZK4aoXFrRNEAEE1
+l4y3/9VX/eLf/huTZODyzOltKqqjL3vR6uqhG8pDR/xqj6uSfCnOO/QHtLQyOL2xmQBX9T93/xey
+BiZWy+cJqRPWheFLf4srEAZo4bOVvsoZb7rndTsnTq70qrW1tcQ8OHKNW14tVgYspKkGJfEQB3G2
+vNzr9f3hw6vD0fby8uDYsWPCvZiaF/pjfT9xdQoLM221FquJ9XVrK4f7RzYe36KQnSN2vjYzYgMH
+0eBSlpC5ydIYhyRhnLZHaTtLWD28fNvtL806vorb6iBwL/QNfI84vHZ0iY+efXS0fqNVfRHvQ8yi
+DOGxT7GXVzNbjKoQF9mNt9LmdjqjrvE9f/2N6wBESNUMMpsVdM7703HVCovADG2j5o5cQRWnMjbB
+6lANlirmug4wzuRqUGapSEwjueB8YF+P0/Y47BC7eriLmAuCAdkIMJtGxZis09ZFuWrN+2I8IFkK
+zcQ7X/oCTeobLSkQwlNnz54e1nXoEw6rrExYxr5pyklxpDizcxKp6QFcq42iLoZaRSEKZJvxQn/W
+K5Gr1mLt/5B0ZuPs2a2NngcAjUljYuWY3Klh1oKW19fH4zrZEF5zlVOzmyg7I5d5bbC6s7njgZAV
+MBiBrYu6PyNXrcXSRR+Iabve+fMHv90IsqPterzdDJP47S3U2tu9pjq5pptLslO6nYKHPXdyOKqV
+HBUI1jTN1775DSMhJgAwg3XLOc/MVSsstL1vgMHUlPgbD33HCleHtDsab0zGw6ybE4RyabRUPEmj
+3YKHnsfeBXGb4zqbOCqz2vGTp/78u9/KlkXc9E0VnbaekatTWLq/341pbPG+P/nMY2dONU389oOP
+rF93Qx4MsL6G9bXGey17p5t6mM2Vg/E4nj21xeaGoyZl/vBHf59BEWhi2Hv31hp2wdGLc9VG3vds
+CoGZ1QzMG9tn73zlXVT2bjp2xwglHT4s66vJSYrGY82j4aEetp96bHTiRI+cM/+1b37nQ/f+YdNG
+xRZyA9u33fuxW3s+jytPWLRwtGHOdmHuWcDTdBY+N62FCWYJ+vjxE2fq3dvvvPsl19/qXZ8PL9mg
+16ib7CQMwwq0HG1uPfLdQ1ygjk+cOPs//K//og23O++ZWfXC8YUXWFh0TqPtNcU5hy2ef8BckcI6
+5x9tRPJCbcEXaFLYfDlnFsokJl8WqhmEh7/zyMmNjXvuevXLbrtjTEEFYRRsVLudszf03YlvfUXG
+o4Loi1/80q/+b/9yogDAzMwcY7xoI9IFevFpjsvsnp3XMotZY4v/aRc5/yC4QoS1oJB96S60v4n2
+umb+C7JQCDFrPpnmeS4IK+fcWj4/KB/99kP/9t4/VLYXv+imNV/0QkgnHn3pWmGbT423zmycOfMb
+/+o3PvGpT9UJaiAisGg2ImYQgQh2Tg4hE8TgFhYoz79jATEIbS4XGMRgBk0PAhPxBRO8npHpBXj2
+GNK0NWShWaYnEYFp+iFm6WQyvwkQ72/9S+EKcQ4W5hCk+5/oC04vdN5/c4ygaDvcAQyk+VuRsJnt
+NVoGxEOq3trai44du+GmWwonfQ6bJ5/81re+c/yJJ9uzql5hSk0MAE+N5tTHUsyjrwTMgoFsAKAX
+skgM8OxmFZzb0wCbfYBLWSayRYvemmsD2b6Gax8ypb0rthfl8241XyZ7eoUIax/nS0nPaaXpWQs1
+W5TAM0XqYiFXO5jNhtT2BCJkAAbyqEqwQ6wRa9j02kxw4kJIRVEsjoA07ZZp9+vcnl6oQRf7jAx+
+dk8GSm3UYkFbl2IojLCgZSJjNm3vdXYPpAtX3NfU/54I63w71GKAogApkObNQYDBTTXHClJg2nUL
+BQ/7hUWzQJSgrPrNcLx3YRFongcqvPetpFrH3MyI6MLCAkAC4/YKDCWA99Q/e3vsXTzPAiJzE8Ig
+QC9pdajVphHAMObZfdLsbp/+reeLCZdr+fMFFhYvfJhBr4x1EwzAzF8hqE27ARCQoAhCkAyL08Yi
+oGBqvDfv83gMA8zOraQh9AdL4/EQwjCAjdQsLTTA1JKBHelejjtExLmiaSZEYpZ5z3Cqzn6RwYAz
+sEPhwAXASG1XKcxgNvVuDEAGMrLAc+l2m2FZVN774WhHpABUp4vmz7lz2cCCtvKDmYV64vtNbUXh
+lgaUdJxz1pjMLIaQYAw4YjVz4lRzWlhNvwqFBaBfyE/+xF/5gVtuXh4MlpcHS0tLKaUU49i8HbrF
+Ch5UmwMeFmFH6gajnCc2GTZf/c7D/+b/u3cYW5tADOPWss1iD0VV1XV97Y03vOs/evehQ+uOBJYt
+TMY722XlAdQhDce1EYv4kJIlc86ZmXPua1/7xqc+9SlmcU5i08xaX9uWY4PAeRRvuuctP/nWn7z5
+6E1pPBIFMQPg1toxASqFO3Xm1EOPP3ry9Ikv/NmfPnj8u2NMCJKQqrIKMaimWdjiuXUuW9uQBaP0
+rl/4/i0vedmxl778ZS/7wcGSK/sT01FMSWN6/KFHvvX1b/z51/48hFgUfrse91wxSeE5Xe5ZckUM
+hSzQDBD+87/3n7zxda/veefYrrv+umuOXpuS5RB3qMw3viz1bNWdWOWNlea0DLf07EiH6eQjp1aP
+vuh9f/jpX/knv9F6QzOxcuu8M0AiRVX+9v/7O6+8++6nnnqciFkTYrN14qmq8EaoYzp5dsPg2BUx
+ZlUVETPz3ov4f/Nb7/ujT9xb+rKJ9V6zMWAQRYniL932xl/7n3+tCL5Q78mzTQfQ6Z9sgJa9KmqM
+OQWk4Xj04KOPfPpzn/m/Pvz+E/n0CLvtmxLN7p1sn5v99K1nIPSBQ4d7L73zjh+58657rr/lBq6i
+648hm74YJx0iZo2pIOGsOzs7Dz744Cfuu++R408QcED1uFdEdkOrqqpyL7/ttqXl4tTjj1+zsj7e
+3GxWVmJO1uiEdGe8lTVDTjk6VTanZGMznR7pRr2WrH7q4btffPO1AxwfLZao7hWpZs0vvvXY7T90
+99ntoVLfE4gbopDroJYzI8aUYgQMYE2aLAIumyhztPyjf+XNf/SJj8d2SWcxAGIAIMBdt/7g0Wo5
+NbGiIgdq/a02o0bNDArS8XAMwMQc8Sqv/PCtr7nnxa9610/+1Ic++6H/5bd+dWwhQyMyoAyDXdRq
+7c09AQBizqEgHHrVsbfdecebXnTsDjgZT0bCsSomhY+mE0pBc6KcScoQ8urqkR+8tbj95bd99vOf
+/b2P/R7pfI1q0S24KoTVfhzvyqxJOfSXpM/5uqUVL7SydjgMm+NnziSMFDXrVmHbvh6GM2fiyeAm
+UkQiR8smq0KnYQGgqc3ieVQChmppkIuyOrSS824cj/uVr4fD1aVlJtuJ9SREFoGJJbCpUDQo+cOp
+6Ot6QZL2wpp2TvszAU61yIksMDl1rjU2BCIStH76FAVABsmMzN7k2NLN/+hn/7Mfu/P1/92/+mf3
+ff3TvWownmx6kbIsd8fjubbmkX3eW4Ngo/aJlCN865tf/9dfdssbnV9BUlTj9cMh+R3wEHlX8q6k
+cUo5J23qAK0mIXsaNPXmW378jS+97ZYPfuCDjz15YhJQuoERQhxfFkfrilqEZoCbVFc9xzlTTtLW
+hTIRm2lNOpI8drFOOxMbgRopoqA2NBm1arCFMOk5QVMoITMyg6WXE1uE56Kq+t57IspmClYCG5iI
+WcgV5PqxKG25n6sCzHqhR7ldkWQotyEPSkqaCdODNc+um3luZhhgUYihTG5yfPeVL7rr1//pr733
+HT/fTHaWq57mPBmPL95KZoARSHnAS0s4/O6/+nduPXpnaX3H7H0j5S7cBmSLeUhUiwWxJKZiQOvm
+GwDt9X2K46PXrP78L7zn5luOOkKTRiEOL2NfXikQEZgmTewNVtTI1NgTOydChbBPsUhRtHbZmk0K
+Ow6BYUUKFAPHaClMAw8435oTQAlUg0JRuBxSCqkoisFgpeoN4H0CTZ1+UiZyUrJbtqpvvX5vZbUY
+9OFkGnzcv0rSDiKtbhIhCjIhsbZH3n+oQAUqajR9JYpq5cnJqpT/1X/6X/zDv/5LVAcA7C7sYClb
+EmQxAIKi0OW//c5fun755rXeii8b6W1Lf9tXO+Ad5iHTWFCDFGBYAa1YHUiJauKh98EwcWKra8vv
+ec/P3njjIcdGjMs1L7yChNUSGq3KJVMyU2YmzsRwyC5NfKyLEFywuG1p5LUpkX2KPtQcG6R9DaLn
+NhAlcM1UF54oZcq5FOf7FUqv7ELKxkSOyeCImQYmfS2XtKhcvy++AM9WTGxhTATQRhAIkREEkRFF
+s6QsKbfykpQ5ZU6JU5QU29enJ6TMGkxVVaIekupX3vvLf/dn/uYy2F3Ec59XSpLBgd/+5r92/dpN
+FMVy5HKXq00qtrjYJd5pVcVIlAnmTP1sQUJBNajOOhSHrHF3a7tf9t75197eG/BlXEy/IoQ1nz0R
+SdPkepLW1w6hLWyH9noMazDa8vVIJvHMY6dHZ6ONyjSWnY0o1svRp+hpb3FMF472AmjqnTpskEwc
+w5OUJo7FCNTzKtSkqEbe+eVBz0sBK5L2UrE0uO5G9HupjT2ya+eYYmCdmS4mwFQsOYvORUfZ5+xy
+dpZ9ntut5Cw7m1opl7KzNDuqQQnAJ9PdST/iH/zC3/2RH7rHL+S/touVs88xDbCa5buOveKH736D
+qIPl3jL5wcSvDKU3Ij8EjwW1WBBFzpSiNxVTapcoSBqSBkg5B4ZUZT9Gu+7o0be+5S16+aaIV4Sw
+FuXF7GOEc6UZMVnruDiECo3Uo7AxTNvZp8Jpwank3CfrwQqYo+mqyfzt9sW+ATVMQLVQFlAF9kbw
+Au/YiRGSKhExwxFnLRKqKGV0PhDF1kxNd26YhdGnMVVVIDGiIIo2TuN0HEytuUqSoktzGzYbAac/
+KylmxRk+Q5o8YP8//uP/9sVHbzyvY5hnC5VsOCTVz77jnZzNMcqBcq92SxPqTbiYgGtQICSGkhJS
+kaPTbFmTIoKCIRpiVrUMjaRRBD4GfvnLb7/p5iNXl7CotfFCJMKuadqNEnSmDyWEShoaj0Zn6slG
+EGVR5lxRLjEVVgE4A83WMdphY7ZSTAApKBAFMnUgByJAvKfCSeGl8FGTEsgUYFMxFJGKhl0mno9L
+AhLAtYkDszFx6mMRoiDJolOVEuVEObYHUuacOGfO83NAKgqv0yBC0zTLRfWSa2/4xff8XO9CIcb2
+tLXKvfoVdx5eGoTxjqvUrUT0R1RFLqJyUE7tgpIoM3xsvOUy5z0vE1BT0owckaLP0WsuNYv3/Vfe
+9R/gMqWXXRnCam+FwaZMLqSIkgNnJQiIOJOFnlE4OxydnFBdihYwl80nFBlFEq/OgcWgFw//JLFR
+aeMyq1d1FBmJnWc/YCmZCmYnwmbIWbOJslcCM5Ea5sd8jW8eSTIAU1VF0cQaXYouRBeipCghznyp
+zNjz4glK08GakUBBOUFyNfBN2A3j3Z9+x0/c9pIXT1dOifZ2tTQIEOv0I6/94Tgep9xImfxSsnLI
+5QQ8ZqrbOaBTkDpWB82GCEpm0RBbQ04kMVLMEjNizJpBxjnkO+945bzFLlFeBx7H4gsmK+ynTTtg
+ylWvcExGGgdWrQ4mk/Ggt8occzMh8zQk7CJE2tHgs2MqTSQzTUgaidpDGk3XkHkvoMwAYMqaemmL
+Q67iy4qsnib95aoZrKZoTDuEPkGdK+NkqHCJWYWXBt5ZLLKWs9XKduU/7UvWMUUOnBvOgRQczRJB
+2xgp4KY/mCNAjV1bQmapzTcwaGLNCMTZOKhrTBp12Zz+zDvf/M1/9lADJNVFH6sCrl9dO7y+LkbL
+a4UviXqZKssy8TTxCFCNMcNcSoiTXUU2ZIISJRhikwwwI1ihKkZmlgEWgMQ5wd2vvOurX/1q+1JO
+l9LvVwxMIopKycW0ffakaU1xwrlGrjVO8iTaRC0KocwoIrmGJQiPQGOymi0i24WyT9rsEYaKTpzV
+3oLXxBxBCcZGJawwFSIREHJWRYQZE5s6ZJczZ4O2q8lzn30uLGAWbjBSJSjHzDFLbEfD1lYZa5vI
+oLMVxtmfqhyyNFlClqAc2h9A4Sf+8l86PJileyx8LAWOHDkiUpAX3xNXZSkTuQCqCUlUxVQUlBAn
+MeWsNjGMQcEs2d5iM8McbKZ7UpC20df11dXphfSSQvAviLDOvejU0lsh6laUlibN5MnjK1l9vc1h
+k8M2hdFouDMOMRirSGQO4gLzRDDUODKtcwo50SzIpPuvsWfUSQ0TkobJiBjz8chUaLoKZEZ2sXU6
+esbX9hljM50eqrP1HbPzO6z1I9mIrF1YNOSlwdIdd9zupkmh8yRBNMD1t9zkeqV470riisxrpobY
+prk6xqpomibl+RyPbWrH2wX6+coE7704O6695rpphtelxbNeKIt1/nWZUHKuBlz2MuetMY+DCxMJ
+Qwljik1dhyZYAqkrg7gg7ZSad8km0KApp3RO2lp7DWvD98pmJKJGI3BNDCIBlKFsyNN5NptR2/2z
+e+Ksqhdr44UsTaULpNE9O9QsgxKQiDOghmiWq151zz33JMPS0pItZH5m4NCRI+yEPHEpcMk4GCee
+7k3PgGvqmNI84DJVjCmZts8Mw9xcUqbUKs8MZrq2vnZZCr1fwKFw36UVurlRnzq+HcapoNJpdebR
+U0UKGO4gpWZrOzYpk5P+UmC3HdJGSpspndGwpXEiujMZJ1NiyL5i0tlSoRmA4XBInLPtLi3TaLwr
+7TZHZMvLg2keH5GqqSrUNOfJZLy7u5tT0qwARGYDB55moZZtD53/rfv+Nf1xqiqENqRCnJkzcQaS
+WdrZ2Tl27Njscjy/dE8AL3VOJmaSE8XIE+XAol6EDM04pARA2gGOTMi4PVIyGOeMnBGD5aQpqxLn
+pDBWMMBl0at8z5FrnePnrY8rYhFaAcB5WS6K1UnIVVWV1VpTB6szpEHkPvzZcROs50zGaubLpJYE
+mdAAznNeaAClfbYEUFNoyjlns2w0ATsARG3TTxNEmakdg8wcgNbhBWSqACIGXTB8+Ey7keqs+OO8
+/zBjGLGRGAvQqoqSWTJDThDnHCOlfS50NjQxSeGMKXNSDsYBlIw0BatrTdHOrZExP02zMDKDwVor
+Nc3CUAA8HaGNLpetudwWiy7gdOgFVlhml94735fFctE7tBE5lqvSv2YS3M722MYZW3WVi8kwxETB
+0KgFoiDcMDfCQdpQ4IV7t30xAynGGGNMTRsmZVMiISKaZvTybDlD1RKb8iz+ObtZ07nzfiGeYfAg
+vdjrSkoCciqO4IgE0xhTarz33iGmhnjvHbJiEhrfL9iTcjJOyilTk3JT1+MUoxrNLFwGMqwd5lrF
+tGM9LThbfO7P+1Tx/HV2oBaL99+Wzn3q+XQcNndNjEt1fclNsaPqirLoLU2GT9WFFCmMdkaa17P4
+aMakJpqAyJxZo8EEiWEEtQVHZyFACiCrIhBiNhsZlVkZamwK9plZ26rYpEZIlsBEBs7ZZ+8yiTpA
+zML5+tj7RIRMnAlKbUkPg6ZZyQDYYKxQbhP4DACpkYIzWEkiqIGLkEQULWWAczIiYoEma32v6QcB
+mhh8UWhoNGezDDU2CuOUY1Z1psREOnMTzWA6fXxbswWimY8le2EZ44uq/wUW1uIMnObrpTL7jwwi
+WHRFASCE0HaKYp6NEpLfaYptWB4aXLIVpF6/t7N5Moy3Ib2YJPmSWclZRpNgmZwSgimkjU9C82wj
+kP29XxBE3fDkSHt9LhuiTNZjYyIh0E4zTqAjRY81qVggEJuYFo3STohDTluAOdX63OrPmYtvxEoc
+BWAWK+Ybsk09biMQIZsht+nKNMuxATXgpBzEj+AD+URGuclnNrZ3J6Q5E3E7QBNNd+vtlf7B7zxE
+bE1qHCuUnAqUYazIe+UbRCAGOCSDSevbtUnxRoApkcHU+OkKOPgSEowvn7DmD+9eYV1bVDOYlqVY
+Bihm5X09P896V+M6S52dNs5HpmyIE7WxNTXViQw9I68SzaVEmjS35iErEk/zn/YVki7clxIE4pLz
+mcmimUFLUxMiI2nrfMSULStBYVkTxYimYWZfFKLuaQqYdUFmmRjEcwdnngeqUDCxTosfDWakWRJR
+IgqZE7lI0pioQhJM28FP1UynZRgzmiY+dfz48ZMnr1lfTbFmNTYxU1gBi3sPFs3ngzAlNeh8wLBZ
+sy+aqMtqrnCZfaz5/j4G5sK59aq89ujR6w8dOtpfOuLLdXIrlpFzZuYLeV1QStHVwddJkhKHoYYd
+P9ruj3YKhgegTNlMCe34ZQQltZlXrXS+D3TR9iLjqVOzb8hmIkmxiamJKaYYMauJmOZjXfBz096f
+81cWisMUgJIq6/zMdlUnczLOkAwxcwwmkCg5nYbsmeb1zTPrqMD2cPSFL34BIgBmDvh+0S+oxMwU
+WTXub4oF15d0Hs/bV4y5d9rz4fJYrD15GgAcOrR+67Hb77779ceO/eBgqdfUaTyaNE3zhS997utf
+/8pjT3zLOZdCOr/bjTUjqHCEGlwYS56U45pGKrbUywyzrKRtJfG8LdvlarWLxl/svPhW65vP5koL
+VlMJxikH1pBTiZTbSMS+t7hQobYSnuYpNVZr17dJlaZGhKFKClHjrM7gjMTFSBlOVYy0jX235dxm
+RmDAvIjl/MADD7z1zT/Os7UragvlZpDN6k7N1LLq7JGYVsfOTsLC5GmhES6LJC5JWIsN6QjOi1p+
+wxte99Yf/w9TICfLJZBHdSnFYPmIrBd3vueOsxtPffQPPvCHH/9ogPZcFVLMU89AY4xN01TLhZqm
+ZCm7rY06b4zPktNrDmdXKJtieqTplmqWjUgRQrK8b6+9aR3oNBgNItre3h6Nq0GvN715Q85TKxRC
+vXRkrdltTIWIl/q9hqxpmhTy0vI6EU3j0AseLs9SG6bpvsQ6i5LTLFTe1o1ORcyJbOpJGzIotwOi
+OWRn4mDCcBKavDNJjdL21u5UlLMZQFu4E3MW4KHHn7r/85//kde9VlPMObEwskKJzVQt6zyeqoZE
+s36ifdGEc4xc++fMepFeYuj98gyFBAhDc/5b//FPv+un3t7z1vPoFdbzXJiViSstquybjXq9WPmF
+d/+Nv/9Lf+9wbxWZPBcAgQkgIRawQFhdaKiexNy4OhYTN6irQXSSGZEswxKsjS/M+3JjY+Pc3dbm
+/X7xWEC7+2Nd10VRzH6jFYYST2PxeiGLxU8bXWjH6HOWhWya/K5GaqQ6+1NdNm8qBkEC1VljQkhZ
+qV0qZp3GxPcuWZWVAR/92Mc2NjabGOuYNLcP3F5wNuWcco4pCbdKR7vM0I7DUwG1EVQo2fx/F9cx
+L4nLIKz2KQgZv/zed//Ym17TL3UwwGBg/YpWKjlS9Y9UK4eL1SPF2sDKJfR0t/nx1/3oz//cz9cW
+aq1FSudKuNI7V0jR08o1RRrbcDfHhHG23cI1gyoIt2lPrapsdpBwURTHjx+/sHz2vK4Ft6MdYpiE
+PZEMR6OyLAFt6wEdC3EmsnmiLjHDLuZg7Qup2MWdsNapUlLlpHvaSubVJMNBhUKKTcoh5Rimjjbx
+dOvTRVKMDJzc2PnAhz6YkpZlmVWzXsCFMsuwTJbI2vWumbZMyWDWitzIMnR2XHyjg+fE8x8K55d3
+YIW++2fe8o6fftvWxsakrqUsQfBkZYHD1VrPVoRXPJe0dHh3cjYVk82drTe85vWnN0fv+53fyjlC
+GaYxpRwVpTPNIeRxnQqjceFT31MPUZNSatME2vrieY+Kd2c3Ny58l7PBi42dslP2GaqaLRglFYNR
+mEyqwvFC5SbbdLcHNmWOoACaJQsby3mGUOEUzqxQcCaT2WyQ2p1gDGY6S21ol1kyUyJrq3oSSc6c
+iCmYhhRzRs7aWrjFzZLmPd1oBlASvvzlr6+vrL7jbW8HOFui2QISgAtEEUhhtLgSsFgVu7gpwTlT
+gef3zZ/PRVj7Yo97r1XiqbR/8F//yurawD20tDumM7ujI0eOvOjaI9f4Q0uj9SosSS5InamUqWn8
+JvnqVOS7f+jVH/n9j26OnyJzNtsbKDEHy1nRwB4dj9mx9hhUq9TtUz697uweTO3kyZNfeuABuL2p
+P9Gs1YgAcU7Z2OViFUv9TKSh6PdIJjXnzXF0ngoUjHo0GhNVBCJ4hgisWGK4rUlzAhbapCWZffrF
+PZm965V+dUyj8WTSq1xCcor24gQoDMRmliknzoSkGhShykFQlxyMQ/IJTOo55jyZxKScnRvGnemm
+SAvWo90RSWFmSMB9f/y5Oqd3vu1tKWXLykTifGqathoFC7OKvZV1a6Wzl8OxuJnPeDRug6t5tjBk
+z2tce55D4fwBMgBqd995x3LhEMOgv9pfvoaXDq1cf9Pa0WvK5Z5Oc4sTIw36cmi9OrTe6w0ckZHa
+K2673WGJWUDsiFlYzdr9qDLx0Ltmpad9FylkTsppenGbOsVscE42z2yMh8OnqRUP0Zi9Q1WPldWx
+srYFWNAm1pZTvywWTm9THlQsOQTT0frhJXhmZtlvOeaesIhPOQm79cNH4UpyJbvScSlSsiudlMyO
+XSFt0XJBKAxFW9xGRmpimREZKpRmT200XT50SMQ73ufx8GwLTQX6/UIJ/+6zX/jff/3XR6H2ZTFY
+WU6awQSm55pw0Z68uroqwtn0ErfkuyQfS4EKAss/+qpX73z3u/1RvVQOyuVDWiz51cNYXhmL1qVN
+yhjKJhaTUOz4lehWLMukSWNX4DWvfbUh82xzunZbOwF5IzCPCoS1Kvd8ILtYMzmS+79wP8K5PpAt
+9j8wapqdoA1XDfUnXAYuknhTSpMG2iwvle2vMBtxJs5CWTiKpYLd7tYWUp5tBXOBVecmjJgVFEaj
+LTMzI1VqV+Xa+AVBcow5B8tJNamqZrRVDco+k1MqDYXCKbGxKqk43t7azjnrxcehrXHIimT4xkNP
+/Oo//+ef+MwfP3HyhDnJZsZEIrbgjIIJ7eyIZZruMMtdW2Q4GpqaY7lE9/2Swg1LUlDOQu7FN9y0
++8hTtLTu1q9h5+uEAD8iaUJa8aWa5gQvIdGOuCbQeBR2huMgUh679UVrK4Od8QhQYnLUBgZVCZGR
+l8qm75VNCIDJbAGMaO+BePzxx+6//34ARVWGet83v83T0r2Xjd3J2d2ae4caLmu2SJUXZ0a5Cay5
+X/mG5qnoYCQlCFRgfbf0F1/7RhslMtB8a6vFaOiffP7Tu8Ozy6uHR5NGxJHB52k5fJu3p8gDXyZC
+JGGImHhNRVaBqE3EClNK3H6JSlJTJXVF8eCDDzZ1TLPd4GaXm/o885Wz1tjuTMK9H7/vc5//k3vu
+uefOO16xurpKTMYLdmcxw2L6ZQhqBJq9LoASHnro4aip1+s34+GlGK1LElbSDGQP58gVSYent9dv
+6MW4qZp3R3XZL0z6UpRGrIJCc+HGR64tz57affLMk1EHJCROcm5MGxAzi/MeIrth2NSTSci6XGnl
+ssvMZsnMprnHKUURl0Ps93uf/dzncjIAoQ7z8fmc3esy9OTG6Nff9/8Mlldv/YGbz25uLB+5dr2/
+PSI9+fhTk93txx4dnz556vTxk8aSFOVgxYgV7Hrl5+//d+/7178JYtjUdmQCgNkTrUB66Dtf/y//
+0d9/17vfY3DCjg1sTEmhBlU2gJJyMEptyZcgiWqZVUzLPqNAFI2WxRVnj2/snh6mqN9+7Lu/+8F/
+q3m6cfR84J1PDGy2gjV3AQzY3J18/JOfvu9Tn15dXT167bWrq6tVVS302EKCmvF8oVrYpZxjCCGn
+B778RQN2x0O+tO1BLklYaurAajocDrm/wubIlJCcaqjjsBHiJXbODIqUDVykhmx7vGOOxUO12d2p
+1RpVNba2z+pQjyzWzhqQVAWcTGeCZjybrTB7J44cPfnkiS9+4Ytt0r8rihT2tnqaxZUVBI3mvTzw
+zUd/+R/+Y8q2HdGr0F8/FHK1u7EhqPue1CjEqbdr005jX5UhRldWybRf9pumzpiFK/fGXgXhM/f9
+/mfu/zSadobbeoHThGuebsWWdFqtPf0U5ew+DYitRgTI7UQbziOmqXVqtzda8LQYaBMy9s398iwJ
+Ww1ntrbPbG2b4YK1NrZ/N5tCSuSssNSGsi5FE5dHWDAip5Y3tnfi0SOZHCg51JXFNJyMBlIMlqOX
+xrIgsaYJcoV8ZneX2DuhWuPm5ukYQ7siE1Ksc9yMzaY2tQeED1WlJ2IYaZaFJuqX1Xg0YZGPfOQj
+zWgMBQgpNoR2sf4cbwsQiqog3gnTFp0ETE5vQCtoJEFMVi9s5MeCecxQvEu7QwCTZkyzL5TDdLvJ
+qaogQALCLkCzCjRZbCUDBHrOvD3O9o8kJ04o5STO5ZxaH24SIQ45gbnNWN+3ne5sonfehOV8I/Ms
+zE7IDUACZhazPLdkl6Kw5yksXQhvRNjucKxw2lYkG7PxpLYQaeAGYI2IRJQi1eJYsDGp2TnOjKg7
+OzvzlP/MaKBbqd5EDGKVK8iZkJIpK9rk9Jamif3B4P3vf//XvvSVvTlxgvH+QMi8WbOBmIqKAU3B
+ckbhUEewMUvKOeWF9AXCvNK8mTTtwkav6IXQzFKc9jUFgWAwx8iKws32kJgOWEQk1G5jzEC7Iwhg
+TAZnyKYKWMptzCprWrxCTmBBUbimjs/cywtPU1kVKeccMwzkwHNPa54FZ21y7CwglxNgSnAsMV1C
+zdfzFNaFtN9YAvChj33sTa+4S6wc7drxk/XGUIZpfPhmb6Ubu3Dj0bWd46eFqFw9st2crn3JzaTy
+fRrlL335ASXAQXolmLbr8aakU3lYsl/uF9nVZupUnaqGRIx+bymEUBTlvX9w72fu+xRcuyfk/A4X
+hbUY/CHALDa53eGW2IKCBQYzE7QV0GTcji57bzJfLmvCdFrAmJqk+X5J012F2g1g2tFrfidmZkg2
++8W53qAGtNsuUbvlLhnazEAG2trY2aBW12H6Weic2DouNs41dZh3VptOMvu/RQu3+PM00BXTRb8k
+4bny/K1duy9GgiXYE6ePf/vRh9W7R546vTOOK8tHyt5KKHjIeeRx1ppNSyORCH/y1FY9npRedre2
+T5w+8xff+o54giHHEHKcpFBDg0AluwLEkZFcyi5nxyiKYjyZlGV57733fujDHyYv4p6Txd1buidr
+j+n+DnruFynsfaEBn9dGvD+axfNEAQOdk2y2eG268CpJm4M+2wV3xgVi0d9n3+Z6qWkzPCt2++2P
+/d5LX/uaJ+r6phff6mWlShit9kKfIRjGrMl6rrKNs3jq7FqMLtTM1e/87gfP7oZp1xELkZClnBla
+iO+5ooipSFZlpWTB1EQd8f/9/vd/8hN/LF5yzPlpHIhnM6WhBSMy+5aA837zws/eOSftK5Xd23T9
+ma/+HO72ALV1+d/58mQ3ZOBPH/ru//Hb/+fgmvXD16wfXl9ZWyqLIgONU6RJnMQRIU1OnqxGzZr0
+J9uTT37yU9/87qNeUBSCDBgsZyT1qhW5AflSqUwokvloPmOpPzh54vRv/uvf/OQf/TEJqE2svCL2
+5u24AJcUuF/chCwD3378oWFd33HnXddfewMqPy40K/nanXn0yRsOlWnz8frhv1gnNMPw8Y9/9gMf
+uy8TMmDEBiMnlS9f94Y3jHNc6g9WxWM4WpWiAg2cP3Xi5B98/OO/+4HffezhJwFA0db6ncuizi6y
+pwU9kxhnZcfTofBZSXfxrIt89dz3mvM2H3x+7/G8uUw57wR4WGP3fuDDDz/46H/z3/9Pr37zm8No
+eyXLiSfOLNc1tkdx6/SSl6/+2Vc+9IGPfPuJx+Ms0phTlsLlmByJZFtm10e57EtPZR5Pvv2dh770
+2c/+2QN/2sDC5dnPt+N7weV8plzpEDU7bzHc+OrX/dhb33rby1959vjG0cOHJuOzX33gc9+8//6H
+v/WtZEgwMC1+a5IUcuMNN/3Uu94dhXfryZmTp86ePPH4gw9unTk9HtXT8KF7pv1Pno3Fsmce/Rcn
+hs/GV7CFa+2LLX2/jNQXeWIvxU+6zB+9ZEnicmwggrKP8QRQcA8WYYmZy7KcTNoXZea92nzmu7Sy
+NtzZgivAgljDrLdUWdZ6EgCsrPZ3tsdPd/lOWM+PK0JYT/8bNpuk06zMaH7TF8yh3jczmpXhTrfa
+xIHMg56FsBZ5ro37fRYVuOTPezEudyX0NN49L0PV59qRwLOerndcwVwRm4Ls5/v9me8Arqgd/Tqu
+Jl5oi3WxgfJyV3x3fI/pLFbHgfBCW6zz6WzVVUFnsToOhE5YHQdCJ6yOA6ETVseB0Amr40DohNVx
+IHTC6jgQOmF1HAidsDoOhO9BYsr3t3a/v+/+haNrt44DoRNWx4HQCavjQOiE1XEgdMLqOBA6YXUc
+CJ2wOg6ETlgdB0InrI4DoRNWx4HQCavjQOiE1XEgdMLqOBA6YXUcCJ2wOg6ETlgdB0InrI4DoRNW
+x4HQCavjQOiE1XEgdMLqOBA6YXUcCJ2wOg6ETlgdB0InrI4DoRNWx4HQCavjQOiE1XEgdMLqOBA6
+YXUcCJ2wOg6ETlgdB0InrI4DoRNWx4HQCavjQOiE1XEgdMLqOBA6YXUcCJ2wOg6ETlgdB0InrI4D
+oRNWx4HQCavjQOiE1XEgdMLqOBA6YXUcCJ2wOg6ETlgdB0InrI4DoRNWx4HQCavjQOiE1XEgdMLq
+OBA6YXUcCJ2wOg6E/x8C66gpbhsR8wAAACV0RVh0ZGF0ZTpjcmVhdGUAMjAyNC0wNS0yOVQwODoy
+MjozNSswMDowML3ddIQAAAAldEVYdGRhdGU6bW9kaWZ5ADIwMjQtMDUtMjlUMDg6MjI6MzUrMDA6
+MDDMgMw4AAAAKHRFWHRkYXRlOnRpbWVzdGFtcAAyMDI0LTA1LTI5VDA4OjIyOjM1KzAwOjAwm5Xt
+5wAAABh0RVh0cGRmOkF1dGhvcgBKaW5hIERlc2lnbmVyjkIguAAAABV0RVh0eG1wOkNyZWF0b3JU
+b29sAENhbnZh6scSsQAAAABJRU5ErkJggg==" />
+</svg>

web/src/assets/svg/llm/volc_engine.svg

@@ -0,0 +1,14 @@
+<svg width="200" height="200" viewBox="0 0 20 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_4967_21175)">
+<path d="M20.511 15.3019L17.2442 28.1928C17.2362 28.2282 17.2364 28.2649 17.2447 28.3001C17.2531 28.3354 17.2694 28.3683 17.2923 28.3964C17.3153 28.4244 17.3444 28.4468 17.3773 28.4619C17.4103 28.477 17.4462 28.4844 17.4825 28.4835H24.0137C24.0499 28.4844 24.0859 28.477 24.1188 28.4619C24.1518 28.4468 24.1809 28.4244 24.2038 28.3964C24.2268 28.3683 24.2431 28.3354 24.2514 28.3001C24.2598 28.2649 24.26 28.2282 24.252 28.1928L20.9685 15.3019C20.9541 15.2524 20.924 15.209 20.8827 15.178C20.8415 15.1471 20.7913 15.1304 20.7397 15.1304C20.6882 15.1304 20.638 15.1471 20.5968 15.178C20.5555 15.209 20.5254 15.2524 20.511 15.3019V15.3019Z" fill="#00E5E5"/>
+<path d="M2.53051 18.2228L-5.28338e-06 28.1924C-0.00799016 28.2277 -0.00780431 28.2644 0.000538111 28.2997C0.00888053 28.335 0.0251596 28.3679 0.0481365 28.3959C0.0711133 28.4239 0.100182 28.4464 0.133131 28.4615C0.166079 28.4766 0.202039 28.484 0.238273 28.4831H5.28025C5.31649 28.484 5.35245 28.4766 5.38539 28.4615C5.41834 28.4464 5.44741 28.4239 5.47039 28.3959C5.49336 28.3679 5.50964 28.335 5.51799 28.2997C5.52633 28.2644 5.52651 28.2277 5.51853 28.1924L2.98563 18.2228C2.97054 18.1742 2.94032 18.1318 2.89938 18.1016C2.85844 18.0714 2.80892 18.0552 2.75807 18.0552C2.70722 18.0552 2.6577 18.0714 2.61676 18.1016C2.57582 18.1318 2.5456 18.1742 2.53051 18.2228V18.2228Z" fill="#00E5E5"/>
+<path d="M6.99344 9.96839L2.38275 28.1919C2.37498 28.2263 2.37494 28.262 2.38262 28.2964C2.3903 28.3308 2.40552 28.363 2.42717 28.3908C2.44882 28.4186 2.47637 28.4413 2.50783 28.4572C2.53929 28.473 2.57388 28.4817 2.60911 28.4826H11.8329C11.8691 28.4835 11.9051 28.4761 11.938 28.461C11.971 28.4459 12 28.4235 12.023 28.3955C12.046 28.3675 12.0623 28.3345 12.0706 28.2993C12.079 28.264 12.0791 28.2273 12.0712 28.1919L7.44855 9.96839C7.43347 9.91982 7.40325 9.87736 7.36231 9.8472C7.32136 9.81705 7.27185 9.80078 7.221 9.80078C7.17015 9.80078 7.12063 9.81705 7.07969 9.8472C7.03874 9.87736 7.00852 9.91982 6.99344 9.96839Z" fill="#006EFF"/>
+<path d="M14.9472 4.17346C14.9321 4.1249 14.9019 4.08244 14.861 4.05228C14.82 4.02213 14.7705 4.00586 14.7197 4.00586C14.6688 4.00586 14.6193 4.02213 14.5784 4.05228C14.5374 4.08244 14.5072 4.1249 14.4921 4.17346L8.18963 28.192C8.18165 28.2273 8.18183 28.264 8.19017 28.2993C8.19852 28.3346 8.2148 28.3675 8.23777 28.3955C8.26075 28.4235 8.28982 28.446 8.32277 28.4611C8.35572 28.4762 8.39168 28.4835 8.42791 28.4827H21.0233C21.0596 28.4835 21.0955 28.4762 21.1285 28.4611C21.1614 28.446 21.1905 28.4235 21.2135 28.3955C21.2364 28.3675 21.2527 28.3346 21.2611 28.2993C21.2694 28.264 21.2696 28.2273 21.2616 28.192L14.9472 4.17346Z" fill="#006EFF"/>
+<path d="M10.3175 12.6188L6.31915 28.1903C6.31074 28.2258 6.31061 28.2628 6.31875 28.2984C6.3269 28.3339 6.34311 28.3672 6.36614 28.3955C6.38916 28.4238 6.41839 28.4465 6.45155 28.4617C6.48472 28.4769 6.52094 28.4844 6.55743 28.4834H14.535C14.5715 28.4844 14.6077 28.4769 14.6409 28.4617C14.674 28.4465 14.7033 28.4238 14.7263 28.3955C14.7493 28.3672 14.7655 28.3339 14.7737 28.2984C14.7818 28.2628 14.7817 28.2258 14.7733 28.1903L10.7726 12.6188C10.7575 12.5702 10.7273 12.5278 10.6863 12.4976C10.6454 12.4674 10.5959 12.4512 10.545 12.4512C10.4942 12.4512 10.4447 12.4674 10.4037 12.4976C10.3628 12.5278 10.3326 12.5702 10.3175 12.6188Z" fill="#00E5E5"/>
+</g>
+<defs>
+<clipPath id="clip0_4967_21175">
+<rect width="99.9412" height="24.5665" fill="white" transform="translate(0 4)"/>
+</clipPath>
+</defs>
+</svg>

web/src/components/chunk-method-modal/index.tsx

@@ -23,6 +23,9 @@ import { useFetchParserListOnMount } from './hooks';
 
 import { useTranslate } from '@/hooks/commonHooks';
 import LayoutRecognize from '../layout-recognize';
+import ParseConfiguration, {
+  showRaptorParseConfiguration,
+} from '../parse-configuration';
 import styles from './index.less';
 
 interface IProps extends Omit<IModalManagerChildrenProps, 'showModal'> {
@@ -111,6 +114,7 @@ const ChunkMethodModal: React.FC<IProps> = ({
       onCancel={hideModal}
       afterClose={afterClose}
       confirmLoading={loading}
+      width={700}
     >
       <Space size={[0, 8]} wrap>
         <Form.Item label={t('chunkMethod')} className={styles.chunkMethod}>
@@ -255,6 +259,9 @@ const ChunkMethodModal: React.FC<IProps> = ({
           </Form.Item>
         )}
         {showMaxTokenNumber && <MaxTokenNumber></MaxTokenNumber>}
+        {showRaptorParseConfiguration(selectedTag) && (
+          <ParseConfiguration></ParseConfiguration>
+        )}
       </Form>
     </Modal>
   );

web/src/components/parse-configuration/index.tsx

@@ -0,0 +1,206 @@
+import { useTranslate } from '@/hooks/commonHooks';
+import { PlusOutlined } from '@ant-design/icons';
+import {
+  Button,
+  Divider,
+  Flex,
+  Form,
+  Input,
+  InputNumber,
+  Slider,
+  Switch,
+} from 'antd';
+import random from 'lodash/random';
+
+export const excludedParseMethods = ['table', 'resume', 'one',"picture"];
+
+export const showRaptorParseConfiguration = (parserId: string) => {
+  return !excludedParseMethods.includes(parserId);
+};
+
+// The three types "table", "resume" and "one" do not display this configuration.
+const ParseConfiguration = () => {
+  const form = Form.useFormInstance();
+  const { t } = useTranslate('knowledgeConfiguration');
+
+  const handleGenerate = () => {
+    form.setFieldValue(
+      ['parser_config', 'raptor', 'random_seed'],
+      random(10000),
+    );
+  };
+
+  return (
+    <>
+      <Divider></Divider>
+      <Form.Item
+        name={['parser_config', 'raptor', 'use_raptor']}
+        label={t('useRaptor')}
+        initialValue={false}
+        valuePropName="checked"
+        tooltip={t('useRaptorTip')}
+      >
+        <Switch />
+      </Form.Item>
+      <Form.Item
+        shouldUpdate={(prevValues, curValues) =>
+          prevValues.parser_config.raptor.use_raptor !==
+          curValues.parser_config.raptor.use_raptor
+        }
+      >
+        {({ getFieldValue }) => {
+          const useRaptor = getFieldValue([
+            'parser_config',
+            'raptor',
+            'use_raptor',
+          ]);
+
+          return (
+            useRaptor && (
+              <>
+                <Form.Item
+                  name={['parser_config', 'raptor', 'prompt']}
+                  label={t('prompt')}
+                  initialValue={t('promptText')}
+                  tooltip={t('promptTip')}
+                  rules={[
+                    {
+                      required: true,
+                      message: t('promptMessage'),
+                    },
+                  ]}
+                >
+                  <Input.TextArea rows={8} />
+                </Form.Item>
+                <Form.Item label={t('maxToken')} tooltip={t('maxTokenTip')}>
+                  <Flex gap={20} align="center">
+                    <Flex flex={1}>
+                      <Form.Item
+                        name={['parser_config', 'raptor', 'max_token']}
+                        noStyle
+                        initialValue={256}
+                        rules={[
+                          {
+                            required: true,
+                            message: t('maxTokenMessage'),
+                          },
+                        ]}
+                      >
+                        <Slider max={2048} style={{ width: '100%' }} />
+                      </Form.Item>
+                    </Flex>
+                    <Form.Item
+                      name={['parser_config', 'raptor', 'max_token']}
+                      noStyle
+                      rules={[
+                        {
+                          required: true,
+                          message: t('maxTokenMessage'),
+                        },
+                      ]}
+                    >
+                      <InputNumber max={2048} min={0} />
+                    </Form.Item>
+                  </Flex>
+                </Form.Item>
+                <Form.Item label={t('threshold')} tooltip={t('thresholdTip')}>
+                  <Flex gap={20} align="center">
+                    <Flex flex={1}>
+                      <Form.Item
+                        name={['parser_config', 'raptor', 'threshold']}
+                        noStyle
+                        initialValue={0.1}
+                        rules={[
+                          {
+                            required: true,
+                            message: t('thresholdMessage'),
+                          },
+                        ]}
+                      >
+                        <Slider
+                          min={0}
+                          max={1}
+                          style={{ width: '100%' }}
+                          step={0.01}
+                        />
+                      </Form.Item>
+                    </Flex>
+                    <Form.Item
+                      name={['parser_config', 'raptor', 'threshold']}
+                      noStyle
+                      rules={[
+                        {
+                          required: true,
+                          message: t('thresholdMessage'),
+                        },
+                      ]}
+                    >
+                      <InputNumber max={1} min={0} step={0.01} />
+                    </Form.Item>
+                  </Flex>
+                </Form.Item>
+                <Form.Item label={t('maxCluster')} tooltip={t('maxClusterTip')}>
+                  <Flex gap={20} align="center">
+                    <Flex flex={1}>
+                      <Form.Item
+                        name={['parser_config', 'raptor', 'max_cluster']}
+                        noStyle
+                        initialValue={64}
+                        rules={[
+                          {
+                            required: true,
+                            message: t('maxClusterMessage'),
+                          },
+                        ]}
+                      >
+                        <Slider min={1} max={1024} style={{ width: '100%' }} />
+                      </Form.Item>
+                    </Flex>
+                    <Form.Item
+                      name={['parser_config', 'raptor', 'max_cluster']}
+                      noStyle
+                      rules={[
+                        {
+                          required: true,
+                          message: t('maxClusterMessage'),
+                        },
+                      ]}
+                    >
+                      <InputNumber max={1024} min={1} />
+                    </Form.Item>
+                  </Flex>
+                </Form.Item>
+                <Form.Item label={t('randomSeed')}>
+                  <Flex gap={20} align="center">
+                    <Flex flex={1}>
+                      <Form.Item
+                        name={['parser_config', 'raptor', 'random_seed']}
+                        noStyle
+                        initialValue={0}
+                        rules={[
+                          {
+                            required: true,
+                            message: t('randomSeedMessage'),
+                          },
+                        ]}
+                      >
+                        <InputNumber style={{ width: '100%' }} />
+                      </Form.Item>
+                    </Flex>
+                    <Form.Item noStyle>
+                      <Button type="primary" onClick={handleGenerate}>
+                        <PlusOutlined />
+                      </Button>
+                    </Form.Item>
+                  </Flex>
+                </Form.Item>
+              </>
+            )
+          );
+        }}
+      </Form.Item>
+    </>
+  );
+};
+
+export default ParseConfiguration;

web/src/components/rerank.tsx

@@ -0,0 +1,57 @@
+import { LlmModelType } from '@/constants/knowledge';
+import { useTranslate } from '@/hooks/commonHooks';
+import { useSelectLlmOptionsByModelType } from '@/hooks/llmHooks';
+import { Form, Select, Slider } from 'antd';
+
+type FieldType = {
+  rerank_id?: string;
+  top_k?: number;
+};
+
+export const RerankItem = () => {
+  const { t } = useTranslate('knowledgeDetails');
+  const allOptions = useSelectLlmOptionsByModelType();
+
+  return (
+    <Form.Item
+      label={t('rerankModel')}
+      name={'rerank_id'}
+      tooltip={t('rerankTip')}
+    >
+      <Select
+        options={allOptions[LlmModelType.Rerank]}
+        allowClear
+        placeholder={t('rerankPlaceholder')}
+      />
+    </Form.Item>
+  );
+};
+
+const Rerank = () => {
+  const { t } = useTranslate('knowledgeDetails');
+
+  return (
+    <>
+      <RerankItem></RerankItem>
+      <Form.Item noStyle dependencies={['rerank_id']}>
+        {({ getFieldValue }) => {
+          const rerankId = getFieldValue('rerank_id');
+          return (
+            rerankId && (
+              <Form.Item<FieldType>
+                label={t('topK')}
+                name={'top_k'}
+                initialValue={1024}
+                tooltip={t('topKTip')}
+              >
+                <Slider max={2048} min={1} />
+              </Form.Item>
+            )
+          );
+        }}
+      </Form.Item>
+    </>
+  );
+};
+
+export default Rerank;

web/src/components/similarity-slider/index.tsx

@@ -26,7 +26,7 @@ const SimilaritySlider = ({ isTooltipShown = false }: IProps) => {
       <Form.Item<FieldType>
         label={t('vectorSimilarityWeight')}
         name={'vector_similarity_weight'}
-        initialValue={0.3}
+        initialValue={1 - 0.3}
         tooltip={isTooltipShown && t('vectorSimilarityWeightTip')}
       >
         <Slider max={1} step={0.01} />

web/src/constants/knowledge.ts

@@ -47,6 +47,7 @@ export enum LlmModelType {
   Chat = 'chat',
   Image2text = 'image2text',
   Speech2text = 'speech2text',
+  Rerank = 'rerank',
 }
 
 export enum KnowledgeSearchParams {

web/src/hooks/llmHooks.ts

@@ -92,6 +92,7 @@ export const useSelectLlmOptionsByModelType = () => {
     [LlmModelType.Speech2text]: groupOptionsByModelType(
       LlmModelType.Speech2text,
     ),
+    [LlmModelType.Rerank]: groupOptionsByModelType(LlmModelType.Rerank),
   };
 };
 

web/src/interfaces/database/chat.ts

@@ -47,6 +47,8 @@ export interface IDialog {
   tenant_id: string;
   update_date: string;
   update_time: number;
+  vector_similarity_weight: number;
+  similarity_threshold: number;
 }
 
 export interface IConversation {

web/src/interfaces/database/flow.ts

@@ -0,0 +1,19 @@
+export type DSLComponents = Record<string, Operator>;
+
+export interface DSL {
+  components: DSLComponents;
+  history: any[];
+  path: string[];
+  answer: any[];
+}
+
+export interface Operator {
+  obj: OperatorNode;
+  downstream: string[];
+  upstream: string[];
+}
+
+export interface OperatorNode {
+  component_name: string;
+  params: Record<string, unknown>;
+}

web/src/interfaces/database/knowledge.ts

@@ -78,7 +78,7 @@ export interface IChunk {
   chunk_id: string;
   content_with_weight: string;
   doc_id: string;
-  docnm_kwd: string;
+  doc_name: string;
   img_id: string;
   important_kwd: any[];
   positions: number[][];
@@ -89,7 +89,7 @@ export interface ITestingChunk {
   content_ltks: string;
   content_with_weight: string;
   doc_id: string;
-  docnm_kwd: string;
+  doc_name: string;
   img_id: string;
   important_kwd: any[];
   kb_id: string;

web/src/locales/en.ts

@@ -93,15 +93,12 @@ export default {
       progressMsg: 'Progress Msg',
       testingDescription:
         'Final step! After success, leave the rest to Infiniflow AI.',
-      topK: 'Top K',
-      topKTip:
-        "For the computaion cost, not all the retrieved chunk will be computed vector cosine similarity with query. The bigger the 'Top K' is, the higher the recall rate is, the slower the retrieval speed is.",
       similarityThreshold: 'Similarity threshold',
       similarityThresholdTip:
         "We use hybrid similarity score to evaluate distance between two lines of text. It's weighted keywords similarity and vector cosine similarity. If the similarity between query and chunk is less than this threshold, the chunk will be filtered out.",
-      vectorSimilarityWeight: 'Vector similarity weight',
+      vectorSimilarityWeight: 'Keywords similarity weight',
       vectorSimilarityWeightTip:
-        "We use hybrid similarity score to evaluate distance between two lines of text. It's weighted keywords similarity and vector cosine similarity. The sum of both weights is 1.0.",
+        " We use hybrid similarity score to evaluate distance between two lines of text. It's weighted keywords similarity and vector cosine similarity or rerank score(0~1). The sum of both weights is 1.0.",
       testText: 'Test text',
       testTextPlaceholder: 'Please input your question!',
       testingLabel: 'Testing',
@@ -143,6 +140,11 @@ export default {
       chunk: 'Chunk',
       bulk: 'Bulk',
       cancel: 'Cancel',
+      rerankModel: 'Rerank Model',
+      rerankPlaceholder: 'Please select',
+      rerankTip: `If it's empty. It uses embeddings of query and chunks to compuste vector cosine similarity. Otherwise, it uses rerank score in place of  vector cosine similarity.`,
+      topK: 'Top-K',
+      topKTip: `K chunks will be fed into rerank models.`,
     },
     knowledgeConfiguration: {
       titleDescription:
@@ -265,6 +267,26 @@ export default {
     </p><p>
     If you want to summarize something that needs all the context of an article and the selected LLM's context length covers the document length, you can try this method.
     </p>`,
+      useRaptor: 'Use RAPTOR to enhance retrieval',
+      useRaptorTip:
+        'Recursive Abstractive Processing for Tree-Organized Retrieval, please refer to https://huggingface.co/papers/2401.18059',
+      prompt: 'Prompt',
+      promptTip: 'LLM prompt used for summarization.',
+      promptMessage: 'Prompt is required',
+      promptText: `Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:
+      {cluster_content}
+The above is the content you need to summarize.`,
+      maxToken: 'Max token',
+      maxTokenTip: 'Maximum token number for summarization.',
+      maxTokenMessage: 'Max token is required',
+      threshold: 'Threshold',
+      thresholdTip: 'The bigger the threshold is the less cluster will be.',
+      thresholdMessage: 'Threshold is required',
+      maxCluster: 'Max cluster',
+      maxClusterTip: 'Maximum cluster number.',
+      maxClusterMessage: 'Max cluster is required',
+      randomSeed: 'Random seed',
+      randomSeedMessage: 'Random seed is required',
     },
     chunk: {
       chunk: 'Chunk',
@@ -445,6 +467,8 @@ export default {
       sequence2txtModel: 'Sequence2txt model',
       sequence2txtModelTip:
         'The default ASR model all the newly created knowledgebase will use. Use this model to translate voices to corresponding text.',
+      rerankModel: 'Rerank Model',
+      rerankModelTip: `The default rerank model is used to rerank chunks retrieved by users' questions.`,
       workspace: 'Workspace',
       upgrade: 'Upgrade',
       addLlmTitle: 'Add LLM',
@@ -457,6 +481,12 @@ export default {
       baseUrlNameMessage: 'Please input your base url!',
       vision: 'Does it support Vision?',
       ollamaLink: 'How to integrate {{name}}',
+      volcModelNameMessage:
+        'Please input your model name! Format: {"ModelName":"EndpointID"}',
+      addVolcEngineAK: 'VOLC ACCESS_KEY',
+      volcAKMessage: 'Please input your VOLC_ACCESS_KEY',
+      addVolcEngineSK: 'VOLC SECRET_KEY',
+      volcSKMessage: 'Please input your SECRET_KEY',
     },
     message: {
       registered: 'Registered!',

web/src/locales/zh-traditional.ts

@@ -24,7 +24,7 @@ export default {
       copied: '複製成功',
       comingSoon: '即將推出',
       download: '下載',
-      close: '关闭',
+      close: '關閉',
       preview: '預覽',
     },
     login: {
@@ -91,15 +91,12 @@ export default {
       processDuration: '過程持續時間',
       progressMsg: '進度消息',
       testingDescription: '最後一步！成功後，剩下的就交給Infiniflow AI吧。',
-      topK: 'top k',
-      topKTip:
-        '對於計算成本，並非所有檢索到的塊都會計算與查詢的向量餘弦相似度。Top K越大，召回率越高，檢索速度越慢。',
       similarityThreshold: '相似度閾值',
       similarityThresholdTip:
         '我們使用混合相似度得分來評估兩行文本之間的距離。它是加權關鍵詞相似度和向量餘弦相似度。如果查詢和塊之間的相似度小於此閾值，則該塊將被過濾掉。',
-      vectorSimilarityWeight: '向量相似度權重',
+      vectorSimilarityWeight: '關鍵字相似度權重',
       vectorSimilarityWeightTip:
-        '我們使用混合相似度得分來評估兩行文本之間的距離。它是加權關鍵詞相似度和向量餘弦相似度。兩個權重之和為 1.0。',
+        '我們使用混合相似性評分來評估兩行文本之間的距離。它是加權關鍵字相似性和矢量餘弦相似性或rerank得分（0〜1）。兩個權重的總和為1.0。',
       testText: '測試文本',
       testTextPlaceholder: '請輸入您的問題！',
       testingLabel: '測試',
@@ -139,6 +136,11 @@ export default {
       chunk: '解析塊',
       bulk: '批量',
       cancel: '取消',
+      rerankModel: 'rerank模型',
+      rerankPlaceholder: '請選擇',
+      rerankTip: `如果是空的。它使用查詢和塊的嵌入來構成矢量餘弦相似性。否則，它使用rerank評分代替矢量餘弦相似性。`,
+      topK: 'Top-K',
+      topKTip: `K塊將被送入Rerank型號。`,
     },
     knowledgeConfiguration: {
       titleDescription: '在這裡更新您的知識庫詳細信息，尤其是解析方法。',
@@ -238,6 +240,25 @@ export default {
         </p><p>
         如果你要總結的東西需要一篇文章的全部上下文，並且所選LLM的上下文長度覆蓋了文檔長度，你可以嘗試這種方法。
         </p>`,
+      useRaptor: '使用RAPTOR文件增強策略',
+      useRaptorTip: '請參考 https://huggingface.co/papers/2401.18059',
+      prompt: '提示詞',
+      promptMessage: '提示詞是必填項',
+      promptText: `请請總結以下段落。 小心數字，不要編造。 段落如下：
+      {cluster_content}
+以上就是你需要總結的內容。`,
+      maxToken: '最大token數',
+      maxTokenMessage: '最大token數是必填項',
+      threshold: '閾值',
+      thresholdMessage: '閾值是必填項',
+      maxCluster: '最大聚類數',
+      maxClusterMessage: '最大聚類數是必填項',
+      randomSeed: '隨機種子',
+      randomSeedMessage: '隨機種子是必填項',
+      promptTip: 'LLM提示用於總結。',
+      maxTokenTip: '用於匯總的最大token數。',
+      thresholdTip: '閾值越大，聚類越少。',
+      maxClusterTip: '最大聚類數。',
     },
     chunk: {
       chunk: '解析塊',
@@ -410,6 +431,8 @@ export default {
       sequence2txtModel: 'sequence2Txt模型',
       sequence2txtModelTip:
         '所有新創建的知識庫都將使用默認的 ASR 模型。使用此模型將語音翻譯為相應的文本。',
+      rerankModel: 'rerank模型',
+      rerankModelTip: `默認的重讀模型用於用戶問題檢索到重讀塊。`,
       workspace: '工作空間',
       upgrade: '升級',
       addLlmTitle: '添加Llm',
@@ -421,7 +444,12 @@ export default {
       modelNameMessage: '請輸入模型名稱！',
       modelTypeMessage: '請輸入模型類型！',
       baseUrlNameMessage: '請輸入基礎 Url！',
-      ollamaLink: '如何集成Ollama',
+      ollamaLink: '如何集成 {{name}}',
+      volcModelNameMessage: '請輸入模型名稱！格式：{"模型名稱":"EndpointID"}',
+      addVolcEngineAK: '火山 ACCESS_KEY',
+      volcAKMessage: '請輸入VOLC_ACCESS_KEY',
+      addVolcEngineSK: '火山 SECRET_KEY',
+      volcSKMessage: '請輸入VOLC_SECRET_KEY',
     },
     message: {
       registered: '註冊成功',

web/src/locales/zh.ts

@@ -91,15 +91,12 @@ export default {
       processDuration: '过程持续时间',
       progressMsg: '进度消息',
       testingDescription: '最后一步！ 成功后，剩下的就交给Infiniflow AI吧。',
-      topK: 'Top K',
-      topKTip:
-        '对于计算成本，并非所有检索到的块都会计算与查询的向量余弦相似度。 Top K越大，召回率越高，检索速度越慢。',
       similarityThreshold: '相似度阈值',
       similarityThresholdTip:
         '我们使用混合相似度得分来评估两行文本之间的距离。 它是加权关键词相似度和向量余弦相似度。 如果查询和块之间的相似度小于此阈值，则该块将被过滤掉。',
-      vectorSimilarityWeight: '向量相似度权重',
+      vectorSimilarityWeight: '关键字相似度权重',
       vectorSimilarityWeightTip:
-        '我们使用混合相似度得分来评估两行文本之间的距离。 它是加权关键词相似度和向量余弦相似度。 两个权重之和为 1.0。',
+        '我们使用混合相似性评分来评估两行文本之间的距离。它是加权关键字相似性和矢量余弦相似性或rerank得分（0〜1）。两个权重的总和为1.0。',
       testText: '测试文本',
       testTextPlaceholder: '请输入您的问题！',
       testingLabel: '测试',
@@ -140,6 +137,11 @@ export default {
       chunk: '解析块',
       bulk: '批量',
       cancel: '取消',
+      rerankModel: 'Rerank模型',
+      rerankPlaceholder: '请选择',
+      rerankTip: `如果是空的。它使用查询和块的嵌入来构成矢量余弦相似性。否则，它使用rerank评分代替矢量余弦相似性。`,
+      topK: 'Top-K',
+      topKTip: `K块将被送入Rerank型号。`,
     },
     knowledgeConfiguration: {
       titleDescription: '在这里更新您的知识库详细信息，尤其是解析方法。',
@@ -255,6 +257,25 @@ export default {
       </p><p>
       如果你要总结的东西需要一篇文章的全部上下文，并且所选LLM的上下文长度覆盖了文档长度，你可以尝试这种方法。
       </p>`,
+      useRaptor: '使用召回增强RAPTOR策略',
+      useRaptorTip: '请参考 https://huggingface.co/papers/2401.18059',
+      prompt: '提示词',
+      promptMessage: '提示词是必填项',
+      promptText: `请总结以下段落。 小心数字，不要编造。 段落如下：
+      {cluster_content}
+以上就是你需要总结的内容。`,
+      maxToken: '最大token数',
+      maxTokenMessage: '最大token数是必填项',
+      threshold: '阈值',
+      thresholdMessage: '阈值是必填项',
+      maxCluster: '最大聚类数',
+      maxClusterMessage: '最大聚类数是必填项',
+      randomSeed: '随机种子',
+      randomSeedMessage: '随机种子是必填项',
+      promptTip: 'LLM提示用于总结。',
+      maxTokenTip: '用于汇总的最大token数。',
+      thresholdTip: '阈值越大，聚类越少。',
+      maxClusterTip: '最大聚类数。',
     },
     chunk: {
       chunk: '解析块',
@@ -427,6 +448,8 @@ export default {
       sequence2txtModel: 'Sequence2txt模型',
       sequence2txtModelTip:
         '所有新创建的知识库都将使用默认的 ASR 模型。 使用此模型将语音翻译为相应的文本。',
+      rerankModel: 'Rerank模型',
+      rerankModelTip: `默认的重读模型用于用户问题检索到重读块。`,
       workspace: '工作空间',
       upgrade: '升级',
       addLlmTitle: '添加 LLM',
@@ -439,6 +462,11 @@ export default {
       modelTypeMessage: '请输入模型类型！',
       baseUrlNameMessage: '请输入基础 Url！',
       ollamaLink: '如何集成 {{name}}',
+      volcModelNameMessage: '请输入模型名称！格式：{"模型名称":"EndpointID"}',
+      addVolcEngineAK: '火山 ACCESS_KEY',
+      volcAKMessage: '请输入VOLC_ACCESS_KEY',
+      addVolcEngineSK: '火山 SECRET_KEY',
+      volcSKMessage: '请输入VOLC_SECRET_KEY',
     },
     message: {
       registered: '注册成功',

web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx

@@ -8,6 +8,9 @@ import {
 
 import LayoutRecognize from '@/components/layout-recognize';
 import MaxTokenNumber from '@/components/max-token-number';
+import ParseConfiguration, {
+  showRaptorParseConfiguration,
+} from '@/components/parse-configuration';
 import { useTranslate } from '@/hooks/commonHooks';
 import { FormInstance } from 'antd/lib';
 import styles from './index.less';
@@ -99,15 +102,19 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
         {({ getFieldValue }) => {
           const parserId = getFieldValue('parser_id');
 
-          if (parserId === 'naive') {
-            return (
-              <>
-                <MaxTokenNumber></MaxTokenNumber>
-                <LayoutRecognize></LayoutRecognize>
-              </>
-            );
-          }
-          return null;
+          return (
+            <>
+              {parserId === 'naive' && (
+                <>
+                  <MaxTokenNumber></MaxTokenNumber>
+                  <LayoutRecognize></LayoutRecognize>
+                </>
+              )}
+              {showRaptorParseConfiguration(parserId) && (
+                <ParseConfiguration></ParseConfiguration>
+              )}
+            </>
+          );
         }}
       </Form.Item>
 

web/src/pages/add-knowledge/components/knowledge-setting/hooks.ts

@@ -62,7 +62,7 @@ export const useFetchKnowledgeConfigurationOnMount = (form: FormInstance) => {
         'embd_id',
         'parser_id',
         'language',
-        'parser_config.chunk_token_num',
+        'parser_config',
       ]),
       avatar: fileList,
     });

web/src/pages/add-knowledge/components/knowledge-testing/index.tsx

@@ -15,7 +15,10 @@ const KnowledgeTesting = () => {
 
   const handleTesting = async () => {
     const values = await form.validateFields();
-    testChunk(values);
+    testChunk({
+      ...values,
+      vector_similarity_weight: 1 - values.vector_similarity_weight,
+    });
   };
 
   useEffect(() => {

web/src/pages/add-knowledge/components/knowledge-testing/testing-control/index.tsx

@@ -2,8 +2,11 @@ import SimilaritySlider from '@/components/similarity-slider';
 import { Button, Card, Divider, Flex, Form, Input } from 'antd';
 import { FormInstance } from 'antd/lib';
 
+import Rerank from '@/components/rerank';
 import { useTranslate } from '@/hooks/commonHooks';
+import { useFetchLlmList } from '@/hooks/llmHooks';
 import { useOneNamespaceEffectsLoading } from '@/hooks/storeHooks';
+import { useEffect } from 'react';
 import styles from './index.less';
 
 type FieldType = {
@@ -23,6 +26,11 @@ const TestingControl = ({ form, handleTesting }: IProps) => {
     'testDocumentChunk',
   ]);
   const { t } = useTranslate('knowledgeDetails');
+  const fetchLlmList = useFetchLlmList();
+
+  useEffect(() => {
+    fetchLlmList();
+  }, [fetchLlmList]);
 
   const buttonDisabled =
     !question || (typeof question === 'string' && question.trim() === '');
@@ -37,6 +45,7 @@ const TestingControl = ({ form, handleTesting }: IProps) => {
       <section>
         <Form name="testing" layout="vertical" form={form}>
           <SimilaritySlider isTooltipShown></SimilaritySlider>
+          <Rerank></Rerank>
           <Card size="small" title={t('testText')}>
             <Form.Item<FieldType>
               name={'question'}

web/src/pages/chat/chat-configuration-modal/hooks.ts

@@ -1,3 +1,4 @@
+import { useFetchLlmList } from '@/hooks/llmHooks';
 import {
   useFetchTenantInfo,
   useSelectTenantInfo,
@@ -16,3 +17,13 @@ export const useFetchModelId = (visible: boolean) => {
 
   return tenantInfo?.llm_id ?? '';
 };
+
+export const useFetchLlmModelOnVisible = (visible: boolean) => {
+  const fetchLlmList = useFetchLlmList();
+
+  useEffect(() => {
+    if (visible) {
+      fetchLlmList();
+    }
+  }, [fetchLlmList, visible]);
+};

web/src/pages/chat/chat-configuration-modal/index.tsx

@@ -14,7 +14,7 @@ import { variableEnabledFieldMap } from '../constants';
 import { IPromptConfigParameters } from '../interface';
 import { excludeUnEnabledVariables } from '../utils';
 import AssistantSetting from './assistant-setting';
-import { useFetchModelId } from './hooks';
+import { useFetchLlmModelOnVisible, useFetchModelId } from './hooks';
 import ModelSetting from './model-setting';
 import PromptEngine from './prompt-engine';
 
@@ -22,8 +22,8 @@ import { useTranslate } from '@/hooks/commonHooks';
 import styles from './index.less';
 
 const layout = {
-  labelCol: { span: 7 },
-  wrapperCol: { span: 17 },
+  labelCol: { span: 9 },
+  wrapperCol: { span: 15 },
 };
 
 const validateMessages = {
@@ -92,6 +92,7 @@ const ChatConfigurationModal = ({
     const finalValues = {
       dialog_id: initialDialog.id,
       ...nextValues,
+      vector_similarity_weight: 1 - nextValues.vector_similarity_weight,
       prompt_config: {
         ...nextValues.prompt_config,
         parameters: promptEngineRef.current,
@@ -115,6 +116,8 @@ const ChatConfigurationModal = ({
     form.resetFields();
   };
 
+  useFetchLlmModelOnVisible(visible);
+
   const title = (
     <Flex gap={16}>
       <ChatConfigurationAtom></ChatConfigurationAtom>
@@ -142,6 +145,8 @@ const ChatConfigurationModal = ({
           settledModelVariableMap[ModelVariableType.Precise],
         icon: fileList,
         llm_id: initialDialog.llm_id ?? modelId,
+        vector_similarity_weight:
+          1 - (initialDialog.vector_similarity_weight ?? 0.3),
       });
     }
   }, [initialDialog, form, visible, modelId]);

web/src/pages/chat/chat-configuration-modal/model-setting.tsx

@@ -10,7 +10,7 @@ import { useEffect } from 'react';
 import { ISegmentedContentProps } from '../interface';
 
 import { useTranslate } from '@/hooks/commonHooks';
-import { useFetchLlmList, useSelectLlmOptions } from '@/hooks/llmHooks';
+import { useSelectLlmOptionsByModelType } from '@/hooks/llmHooks';
 import { Variable } from '@/interfaces/database/chat';
 import { variableEnabledFieldMap } from '../constants';
 import styles from './index.less';
@@ -30,7 +30,7 @@ const ModelSetting = ({
     value: x,
   }));
 
-  const modelOptions = useSelectLlmOptions();
+  const modelOptions = useSelectLlmOptionsByModelType();
 
   const handleParametersChange = (value: ModelVariableType) => {
     const variable = settledModelVariableMap[value];
@@ -56,8 +56,6 @@ const ModelSetting = ({
     }
   }, [form, initialLlmSetting, visible]);
 
-  useFetchLlmList(LlmModelType.Chat);
-
   return (
     <section
       className={classNames({
@@ -70,7 +68,7 @@ const ModelSetting = ({
         tooltip={t('modelTip')}
         rules={[{ required: true, message: t('modelMessage') }]}
       >
-        <Select options={modelOptions} showSearch />
+        <Select options={modelOptions[LlmModelType.Chat]} showSearch />
       </Form.Item>
       <Divider></Divider>
       <Form.Item

web/src/pages/chat/chat-configuration-modal/prompt-engine.tsx

@@ -29,6 +29,7 @@ import {
 } from '../interface';
 import { EditableCell, EditableRow } from './editable-cell';
 
+import Rerank from '@/components/rerank';
 import { useTranslate } from '@/hooks/commonHooks';
 import { useSelectPromptConfigParameters } from '../hooks';
 import styles from './index.less';
@@ -172,10 +173,10 @@ const PromptEngine = (
       >
         <Slider max={30} />
       </Form.Item>
-
+      <Rerank></Rerank>
       <section className={classNames(styles.variableContainer)}>
         <Row align={'middle'} justify="end">
-          <Col span={7} className={styles.variableAlign}>
+          <Col span={9} className={styles.variableAlign}>
             <label className={styles.variableLabel}>
               {t('variable')}
               <Tooltip title={t('variableTip')}>
@@ -183,7 +184,7 @@ const PromptEngine = (
               </Tooltip>
             </label>
           </Col>
-          <Col span={17} className={styles.variableAlign}>
+          <Col span={15} className={styles.variableAlign}>
             <Button size="small" onClick={handleAdd}>
               {t('add')}
             </Button>

web/src/pages/chat/chat-overview-modal/index.tsx

@@ -1,5 +1,4 @@
 import LineChart from '@/components/line-chart';
-import { Domain } from '@/constants/common';
 import { useSetModalState, useTranslate } from '@/hooks/commonHooks';
 import { IModalProps } from '@/interfaces/common';
 import { IDialog, IStats } from '@/interfaces/database/chat';
@@ -81,8 +80,7 @@ const ChatOverviewModal = ({
             <Flex gap={8} vertical>
               {t('serviceApiEndpoint')}
               <Paragraph copyable className={styles.linkText}>
-                https://
-                {location.hostname === Domain ? Domain : '<YOUR_MACHINE_IP>'}
+                {location.origin}
                 /v1/api/
               </Paragraph>
             </Flex>
@@ -90,7 +88,7 @@ const ChatOverviewModal = ({
               <Button onClick={showApiKeyModal}>{t('apiKey')}</Button>
               <a
                 href={
-                  'https://github.com/infiniflow/ragflow/blob/main/docs/conversation_api.md'
+                  'https://github.com/infiniflow/ragflow/blob/main/docs/references/api.md'
                 }
                 target="_blank"
                 rel="noreferrer"

web/src/pages/chat/embed-modal/index.tsx

@@ -1,6 +1,5 @@
 import CopyToClipboard from '@/components/copy-to-clipboard';
 import HightLightMarkdown from '@/components/highlight-markdown';
-import { Domain } from '@/constants/common';
 import { useTranslate } from '@/hooks/commonHooks';
 import { IModalProps } from '@/interfaces/common';
 import { Card, Modal, Tabs, TabsProps } from 'antd';
@@ -16,7 +15,7 @@ const EmbedModal = ({
   const text = `
   ~~~ html
   <iframe
-  src="https://${Domain}/chat/share?shared_id=${token}"
+  src="${location.origin}/chat/share?shared_id=${token}"
   style="width: 100%; height: 100%; min-height: 600px"
   frameborder="0"
 >

web/src/pages/chat/index.less

@@ -8,6 +8,7 @@
 
     .chatAppContent {
       overflow-y: auto;
+      width: 100%;
     }
 
     .chatAppCard {

web/src/pages/chat/index.tsx

@@ -17,6 +17,7 @@ import {
   Space,
   Spin,
   Tag,
+  Typography,
 } from 'antd';
 import { MenuItemProps } from 'antd/lib/menu/MenuItem';
 import classNames from 'classnames';
@@ -46,6 +47,8 @@ import { IDialog } from '@/interfaces/database/chat';
 import ChatOverviewModal from './chat-overview-modal';
 import styles from './index.less';
 
+const { Text } = Typography;
+
 const Chat = () => {
   const dialogList = useSelectFirstDialogOnMount();
   const { onRemoveDialog } = useDeleteDialog();
@@ -260,7 +263,14 @@ const Chat = () => {
                     <Space size={15}>
                       <Avatar src={x.icon} shape={'square'} />
                       <section>
-                        <b>{x.name}</b>
+                        <b>
+                          <Text
+                            ellipsis={{ tooltip: x.name }}
+                            style={{ width: 130 }}
+                          >
+                            {x.name}
+                          </Text>
+                        </b>
                         <div>{x.description}</div>
                       </section>
                     </Space>
@@ -315,7 +325,14 @@ const Chat = () => {
                   })}
                 >
                   <Flex justify="space-between" align="center">
-                    <div>{x.name}</div>
+                    <div>
+                      <Text
+                        ellipsis={{ tooltip: x.name }}
+                        style={{ width: 150 }}
+                      >
+                        {x.name}
+                      </Text>
+                    </div>
                     {conversationActivated === x.id && x.id !== '' && (
                       <section>
                         <Dropdown

web/src/pages/document-viewer/docx/index.tsx

@@ -5,7 +5,7 @@ import { useFetchDocx } from '../hooks';
 import styles from './index.less';
 
 const Docx = ({ filePath }: { filePath: string }) => {
-  const { succeed, containerRef } = useFetchDocx(filePath);
+  const { succeed, containerRef, error } = useFetchDocx(filePath);
 
   return (
     <>
@@ -16,7 +16,7 @@ const Docx = ({ filePath }: { filePath: string }) => {
           </div>
         </section>
       ) : (
-        <FileError></FileError>
+        <FileError>{error}</FileError>
       )}
     </>
   );

web/src/pages/document-viewer/excel/index.tsx

@@ -3,7 +3,7 @@ import FileError from '../file-error';
 import { useFetchExcel } from '../hooks';
 
 const Excel = ({ filePath }: { filePath: string }) => {
-  const { status, containerRef } = useFetchExcel(filePath);
+  const { status, containerRef, error } = useFetchExcel(filePath);
 
   return (
     <div
@@ -11,7 +11,7 @@ const Excel = ({ filePath }: { filePath: string }) => {
       ref={containerRef}
       style={{ height: '100%', width: '100%' }}
     >
-      {status || <FileError></FileError>}
+      {status || <FileError>{error}</FileError>}
     </div>
   );
 };

web/src/pages/document-viewer/hooks.ts

@@ -3,18 +3,38 @@ import axios from 'axios';
 import mammoth from 'mammoth';
 import { useCallback, useEffect, useRef, useState } from 'react';
 
-const useFetchDocument = () => {
-  const fetchDocument = useCallback((api: string) => {
-    return axios.get(api, { responseType: 'arraybuffer' });
+export const useCatchError = (api: string) => {
+  const [error, setError] = useState('');
+  const fetchDocument = useCallback(async () => {
+    const ret = await axios.get(api);
+    const { data } = ret;
+    if (!(data instanceof ArrayBuffer) && data.retcode !== 0) {
+      setError(data.retmsg);
+    }
+    return ret;
+  }, [api]);
+
+  useEffect(() => {
+    fetchDocument();
+  }, [fetchDocument]);
+
+  return { fetchDocument, error };
+};
+
+export const useFetchDocument = () => {
+  const fetchDocument = useCallback(async (api: string) => {
+    const ret = await axios.get(api, { responseType: 'arraybuffer' });
+    return ret;
   }, []);
 
-  return fetchDocument;
+  return { fetchDocument };
 };
 
 export const useFetchExcel = (filePath: string) => {
   const [status, setStatus] = useState(true);
-  const fetchDocument = useFetchDocument();
+  const { fetchDocument } = useFetchDocument();
   const containerRef = useRef<HTMLDivElement>(null);
+  const { error } = useCatchError(filePath);
 
   const fetchDocumentAsync = useCallback(async () => {
     let myExcelPreviewer;
@@ -39,13 +59,14 @@ export const useFetchExcel = (filePath: string) => {
     fetchDocumentAsync();
   }, [fetchDocumentAsync]);
 
-  return { status, containerRef };
+  return { status, containerRef, error };
 };
 
 export const useFetchDocx = (filePath: string) => {
   const [succeed, setSucceed] = useState(true);
-  const fetchDocument = useFetchDocument();
+  const { fetchDocument } = useFetchDocument();
   const containerRef = useRef<HTMLDivElement>(null);
+  const { error } = useCatchError(filePath);
 
   const fetchDocumentAsync = useCallback(async () => {
     const jsonFile = await fetchDocument(filePath);
@@ -64,9 +85,8 @@ export const useFetchDocx = (filePath: string) => {
           container.innerHTML = docEl.outerHTML;
         }
       })
-      .catch((a) => {
+      .catch(() => {
         setSucceed(false);
-        console.warn('alexei: something went wrong', a);
       });
   }, [filePath, fetchDocument]);
 
@@ -74,5 +94,5 @@ export const useFetchDocx = (filePath: string) => {
     fetchDocumentAsync();
   }, [fetchDocumentAsync]);
 
-  return { succeed, containerRef };
+  return { succeed, containerRef, error };
 };

web/src/pages/document-viewer/pdf/index.tsx

@@ -1,12 +1,14 @@
 import { Skeleton } from 'antd';
 import { PdfHighlighter, PdfLoader } from 'react-pdf-highlighter';
 import FileError from '../file-error';
+import { useCatchError } from '../hooks';
 
 interface IProps {
   url: string;
 }
 
-const DocumentPreviewer = ({ url }: IProps) => {
+const PdfPreviewer = ({ url }: IProps) => {
+  const { error } = useCatchError(url);
   const resetHash = () => {};
 
   return (
@@ -15,7 +17,7 @@ const DocumentPreviewer = ({ url }: IProps) => {
         url={url}
         beforeLoad={<Skeleton active />}
         workerSrc="/pdfjs-dist/pdf.worker.min.js"
-        errorMessage={<FileError></FileError>}
+        errorMessage={<FileError>{error}</FileError>}
         onError={(e) => {
           console.warn(e);
         }}
@@ -40,4 +42,4 @@ const DocumentPreviewer = ({ url }: IProps) => {
   );
 };
 
-export default DocumentPreviewer;
+export default PdfPreviewer;

web/src/pages/flow/canvas/context-menu/index.less

@@ -0,0 +1,18 @@
+.contextMenu {
+  background: white;
+  border-style: solid;
+  box-shadow: 10px 19px 20px rgba(0, 0, 0, 10%);
+  position: absolute;
+  z-index: 10;
+  button {
+    border: none;
+    display: block;
+    padding: 0.5em;
+    text-align: left;
+    width: 100%;
+  }
+
+  button:hover {
+    background: white;
+  }
+}

web/src/pages/flow/canvas/context-menu/index.tsx

@@ -0,0 +1,105 @@
+import { useCallback, useRef, useState } from 'react';
+import { NodeMouseHandler, useReactFlow } from 'reactflow';
+
+import styles from './index.less';
+
+export interface INodeContextMenu {
+  id: string;
+  top: number;
+  left: number;
+  right?: number;
+  bottom?: number;
+  [key: string]: unknown;
+}
+
+export function NodeContextMenu({
+  id,
+  top,
+  left,
+  right,
+  bottom,
+  ...props
+}: INodeContextMenu) {
+  const { getNode, setNodes, addNodes, setEdges } = useReactFlow();
+
+  const duplicateNode = useCallback(() => {
+    const node = getNode(id);
+    const position = {
+      x: node?.position?.x || 0 + 50,
+      y: node?.position?.y || 0 + 50,
+    };
+
+    addNodes({
+      ...(node || {}),
+      data: node?.data,
+      selected: false,
+      dragging: false,
+      id: `${node?.id}-copy`,
+      position,
+    });
+  }, [id, getNode, addNodes]);
+
+  const deleteNode = useCallback(() => {
+    setNodes((nodes) => nodes.filter((node) => node.id !== id));
+    setEdges((edges) => edges.filter((edge) => edge.source !== id));
+  }, [id, setNodes, setEdges]);
+
+  return (
+    <div
+      style={{ top, left, right, bottom }}
+      className={styles.contextMenu}
+      {...props}
+    >
+      <p style={{ margin: '0.5em' }}>
+        <small>node: {id}</small>
+      </p>
+      <button onClick={duplicateNode} type={'button'}>
+        duplicate
+      </button>
+      <button onClick={deleteNode} type={'button'}>
+        delete
+      </button>
+    </div>
+  );
+}
+
+export const useHandleNodeContextMenu = (sideWidth: number) => {
+  const [menu, setMenu] = useState<INodeContextMenu>({} as INodeContextMenu);
+  const ref = useRef<any>(null);
+
+  const onNodeContextMenu: NodeMouseHandler = useCallback(
+    (event, node) => {
+      // Prevent native context menu from showing
+      event.preventDefault();
+
+      // Calculate position of the context menu. We want to make sure it
+      // doesn't get positioned off-screen.
+      const pane = ref.current?.getBoundingClientRect();
+      // setMenu({
+      //   id: node.id,
+      //   top: event.clientY < pane.height - 200 ? event.clientY : 0,
+      //   left: event.clientX < pane.width - 200 ? event.clientX : 0,
+      //   right: event.clientX >= pane.width - 200 ? pane.width - event.clientX : 0,
+      //   bottom:
+      //     event.clientY >= pane.height - 200 ? pane.height - event.clientY : 0,
+      // });
+
+      setMenu({
+        id: node.id,
+        top: event.clientY - 72,
+        left: event.clientX - sideWidth,
+        // top: event.clientY < pane.height - 200 ? event.clientY - 72 : 0,
+        // left: event.clientX < pane.width - 200 ? event.clientX : 0,
+      });
+    },
+    [sideWidth],
+  );
+
+  // Close the context menu if it's open whenever the window is clicked.
+  const onPaneClick = useCallback(
+    () => setMenu({} as INodeContextMenu),
+    [setMenu],
+  );
+
+  return { onNodeContextMenu, menu, onPaneClick, ref };
+};

web/src/pages/flow/canvas/index.tsx

@@ -4,6 +4,7 @@ import ReactFlow, {
   Controls,
   Edge,
   Node,
+  NodeMouseHandler,
   OnConnect,
   OnEdgesChange,
   OnNodesChange,
@@ -13,38 +14,33 @@ import ReactFlow, {
 } from 'reactflow';
 import 'reactflow/dist/style.css';
 
-import { useHandleDrop } from '../hooks';
+import { NodeContextMenu, useHandleNodeContextMenu } from './context-menu';
+
+import FlowDrawer from '../flow-drawer';
+import {
+  useHandleDrop,
+  useHandleKeyUp,
+  useHandleSelectionChange,
+  useShowDrawer,
+} from '../hooks';
+import { dsl } from '../mock';
 import { TextUpdaterNode } from './node';
 
 const nodeTypes = { textUpdater: TextUpdaterNode };
 
-const initialNodes = [
-  {
-    id: 'node-1',
-    type: 'textUpdater',
-    position: { x: 200, y: 50 },
-    data: { value: 123 },
-  },
-  {
-    id: '1',
-    data: { label: 'Hello' },
-    position: { x: 0, y: 0 },
-    type: 'input',
-  },
-  {
-    id: '2',
-    data: { label: 'World' },
-    position: { x: 100, y: 100 },
-  },
-];
+interface IProps {
+  sideWidth: number;
+}
 
-const initialEdges = [
-  { id: '1-2', source: '1', target: '2', label: 'to the', type: 'step' },
-];
+function FlowCanvas({ sideWidth }: IProps) {
+  const [nodes, setNodes] = useState<Node[]>(dsl.graph.nodes);
+  const [edges, setEdges] = useState<Edge[]>(dsl.graph.edges);
 
-function FlowCanvas() {
-  const [nodes, setNodes] = useState<Node[]>(initialNodes);
-  const [edges, setEdges] = useState<Edge[]>(initialEdges);
+  const { selectedEdges, selectedNodes } = useHandleSelectionChange();
+
+  const { ref, menu, onNodeContextMenu, onPaneClick } =
+    useHandleNodeContextMenu(sideWidth);
+  const { drawerVisible, hideDrawer, showDrawer } = useShowDrawer();
 
   const onNodesChange: OnNodesChange = useCallback(
     (changes) => setNodes((nds) => applyNodeChanges(changes, nds)),
@@ -60,7 +56,13 @@ function FlowCanvas() {
     [],
   );
 
-  const { handleDrop, allowDrop } = useHandleDrop(setNodes);
+  const onNodeClick: NodeMouseHandler = useCallback(() => {
+    showDrawer();
+  }, [showDrawer]);
+
+  const { onDrop, onDragOver, setReactFlowInstance } = useHandleDrop(setNodes);
+
+  const { handleKeyUp } = useHandleKeyUp(selectedEdges, selectedNodes);
 
   useEffect(() => {
     console.info('nodes:', nodes);
@@ -68,23 +70,31 @@ function FlowCanvas() {
   }, [nodes, edges]);
 
   return (
-    <div
-      style={{ height: '100%', width: '100%' }}
-      onDrop={handleDrop}
-      onDragOver={allowDrop}
-    >
+    <div style={{ height: '100%', width: '100%' }}>
       <ReactFlow
+        ref={ref}
         nodes={nodes}
         onNodesChange={onNodesChange}
+        onNodeContextMenu={onNodeContextMenu}
         edges={edges}
         onEdgesChange={onEdgesChange}
-        // fitView
+        fitView
         onConnect={onConnect}
         nodeTypes={nodeTypes}
+        onPaneClick={onPaneClick}
+        onDrop={onDrop}
+        onDragOver={onDragOver}
+        onNodeClick={onNodeClick}
+        onInit={setReactFlowInstance}
+        onKeyUp={handleKeyUp}
       >
         <Background />
         <Controls />
+        {Object.keys(menu).length > 0 && (
+          <NodeContextMenu onClick={onPaneClick} {...(menu as any)} />
+        )}
       </ReactFlow>
+      <FlowDrawer visible={drawerVisible} hideModal={hideDrawer}></FlowDrawer>
     </div>
   );
 }