Update version to v0.3.1 (#544)

### What problem does this PR solve?

Update version to v0.3.1

### Type of change

- [x] Documentation Update

Signed-off-by: Jin Hai <haijin.chn@gmail.com>

.gitignore

@@ -21,3 +21,9 @@ Cargo.lock
 
 .idea/
 .vscode/
+
+# Exclude Mac generated files
+.DS_Store
+
+# Exclude the log folder
+docker/ragflow-logs/

Dockerfile.scratch.oc9

@@ -0,0 +1,56 @@
+FROM opencloudos/opencloudos:9.0
+USER root
+
+WORKDIR /ragflow
+
+RUN dnf update -y && dnf install -y wget curl gcc-c++ openmpi-devel
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
+    bash ~/miniconda.sh -b -p /root/miniconda3 && \
+    rm ~/miniconda.sh && ln -s /root/miniconda3/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /root/miniconda3/etc/profile.d/conda.sh" >> ~/.bashrc && \
+    echo "conda activate base" >> ~/.bashrc
+
+ENV PATH /root/miniconda3/bin:$PATH
+
+RUN conda create -y --name py11 python=3.11
+
+ENV CONDA_DEFAULT_ENV py11
+ENV CONDA_PREFIX /root/miniconda3/envs/py11
+ENV PATH $CONDA_PREFIX/bin:$PATH
+
+# RUN curl -sL https://rpm.nodesource.com/setup_14.x | bash -
+RUN dnf install -y nodejs
+
+RUN dnf install -y nginx
+
+ADD ./web ./web
+ADD ./api ./api
+ADD ./conf ./conf
+ADD ./deepdoc ./deepdoc
+ADD ./rag ./rag
+ADD ./requirements.txt ./requirements.txt
+
+RUN dnf install -y openmpi openmpi-devel python3-openmpi
+ENV C_INCLUDE_PATH /usr/include/openmpi-x86_64:$C_INCLUDE_PATH
+ENV LD_LIBRARY_PATH /usr/lib64/openmpi/lib:$LD_LIBRARY_PATH
+RUN rm /root/miniconda3/envs/py11/compiler_compat/ld
+RUN cd ./web && npm i && npm run build
+RUN conda run -n py11 pip install $(grep -ivE "mpi4py" ./requirements.txt) # without mpi4py==3.1.5
+RUN conda run -n py11 pip install redis
+
+RUN dnf update -y && \
+    dnf install -y glib2 mesa-libGL && \
+    dnf clean all
+
+RUN conda run -n py11 pip install ollama
+RUN conda run -n py11 python -m nltk.downloader punkt
+RUN conda run -n py11 python -m nltk.downloader wordnet
+
+ENV PYTHONPATH=/ragflow/
+ENV HF_ENDPOINT=https://hf-mirror.com
+
+ADD docker/entrypoint.sh ./entrypoint.sh
+RUN chmod +x ./entrypoint.sh
+
+ENTRYPOINT ["./entrypoint.sh"]

README.md

@@ -11,11 +11,14 @@
 </p>
 
 <p align="center">
+    <a href="https://github.com/infiniflow/infinity/releases/latest">
+        <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
+    </a>
     <a href="https://demo.ragflow.io" target="_blank">
         <img alt="Static Badge" src="https://img.shields.io/badge/RAGFLOW-LLM-white?&labelColor=dd0af7"></a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v1.0-brightgreen"
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.3.1-brightgreen"
-            alt="docker pull infiniflow/ragflow:v0.3.0"></a>
+            alt="docker pull infiniflow/ragflow:v0.3.1"></a>
       <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
     <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=7d09f1" alt="license">
   </a>
@@ -55,7 +58,7 @@
 
 ## 📌 Latest Features
 
-- 2024-04-19 Support conversation API([detail](./docs/conversation_api.md)).
+- 2024-04-19 Support conversation API ([detail](./docs/conversation_api.md)).
 - 2024-04-16 Add an embedding model 'bce-embedding-base_v1' from [BCEmbedding](https://github.com/netease-youdao/BCEmbedding).
 - 2024-04-16 Add [FastEmbed](https://github.com/qdrant/fastembed), which is designed specifically for light and speedy embedding.
 - 2024-04-11 Support [Xinference](./docs/xinference.md) for local LLM deployment.
@@ -74,7 +77,8 @@
 ### 📝 Prerequisites
 
 - CPU >= 4 cores
-- RAM >= 12 GB
+- RAM >= 16 GB
+- Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
   > If you have not installed Docker on your local machine (Windows, Mac, or Linux), see [Install Docker Engine](https://docs.docker.com/engine/install/).
 
@@ -138,9 +142,10 @@
     * Running on http://x.x.x.x:9380
     INFO:werkzeug:Press CTRL+C to quit
    ```
+   > If you skip this confirmation step and directly log in to RAGFlow, your browser may prompt a `network anomaly` error because, at that moment, your RAGFlow may not be fully initialized.  
 
 5. In your web browser, enter the IP address of your server and log in to RAGFlow.
-   > In the given scenario, you only need to enter `http://IP_OF_YOUR_MACHINE` (**sans** port number) as the default HTTP serving port `80` can be omitted when using the default configurations.
+   > With default settings, you only need to enter `http://IP_OF_YOUR_MACHINE` (**sans** port number) as the default HTTP serving port `80` can be omitted when using the default configurations.
 6. In [service_conf.yaml](./docker/service_conf.yaml), select the desired LLM factory in `user_default_llm` and update the `API_KEY` field with the corresponding API key.
 
    > See [./docs/llm_api_key_setup.md](./docs/llm_api_key_setup.md) for more information.
@@ -174,7 +179,7 @@ To build the Docker images from source:
 ```bash
 $ git clone https://github.com/infiniflow/ragflow.git
 $ cd ragflow/
-$ docker build -t infiniflow/ragflow:v0.3.0 .
+$ docker build -t infiniflow/ragflow:v0.3.1 .
 $ cd ragflow/docker
 $ chmod +x ./entrypoint.sh
 $ docker compose up -d

README_ja.md

@@ -11,11 +11,14 @@
 </p>
 
 <p align="center">
+    <a href="https://github.com/infiniflow/infinity/releases/latest">
+        <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
+    </a>
     <a href="https://demo.ragflow.io" target="_blank">
         <img alt="Static Badge" src="https://img.shields.io/badge/RAGFLOW-LLM-white?&labelColor=dd0af7"></a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v1.0-brightgreen"
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.3.1-brightgreen"
-            alt="docker pull infiniflow/ragflow:v0.3.0"></a>
+            alt="docker pull infiniflow/ragflow:v0.3.1"></a>
       <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
     <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=7d09f1" alt="license">
   </a>
@@ -55,7 +58,7 @@
 
 ## 📌 最新の機能
 
-- 2024-04-19 会話 API をサポートします([詳細](./docs/conversation_api.md))。
+- 2024-04-19 会話 API をサポートします ([詳細](./docs/conversation_api.md))。
 - 2024-04-16 [BCEmbedding](https://github.com/netease-youdao/BCEmbedding) から埋め込みモデル「bce-embedding-base_v1」を追加します。
 - 2024-04-16 [FastEmbed](https://github.com/qdrant/fastembed) は、軽量かつ高速な埋め込み用に設計されています。
 - 2024-04-11 ローカル LLM デプロイメント用に [Xinference](./docs/xinference.md) をサポートします。
@@ -74,7 +77,8 @@
 ### 📝 必要条件
 
 - CPU >= 4 cores
-- RAM >= 12 GB
+- RAM >= 16 GB
+- Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
   > ローカルマシン（Windows、Mac、または Linux）に Docker をインストールしていない場合は、[Docker Engine のインストール](https://docs.docker.com/engine/install/) を参照してください。
 
@@ -138,6 +142,7 @@
     * Running on http://x.x.x.x:9380
     INFO:werkzeug:Press CTRL+C to quit
    ```
+   > もし確認ステップをスキップして直接 RAGFlow にログインした場合、その時点で RAGFlow が完全に初期化されていない可能性があるため、ブラウザーがネットワーク異常エラーを表示するかもしれません。
 
 5. ウェブブラウザで、プロンプトに従ってサーバーの IP アドレスを入力し、RAGFlow にログインします。
    > デフォルトの設定を使用する場合、デフォルトの HTTP サービングポート `80` は省略できるので、与えられたシナリオでは、`http://IP_OF_YOUR_MACHINE`（ポート番号は省略）だけを入力すればよい。
@@ -174,7 +179,7 @@
 ```bash
 $ git clone https://github.com/infiniflow/ragflow.git
 $ cd ragflow/
-$ docker build -t infiniflow/ragflow:v0.3.0 .
+$ docker build -t infiniflow/ragflow:v0.3.1 .
 $ cd ragflow/docker
 $ chmod +x ./entrypoint.sh
 $ docker compose up -d

README_zh.md

@@ -11,11 +11,14 @@
 </p>
 
 <p align="center">
+    <a href="https://github.com/infiniflow/infinity/releases/latest">
+        <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
+    </a>
     <a href="https://demo.ragflow.io" target="_blank">
         <img alt="Static Badge" src="https://img.shields.io/badge/RAGFLOW-LLM-white?&labelColor=dd0af7"></a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v1.0-brightgreen"
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.3.1-brightgreen"
-            alt="docker pull infiniflow/ragflow:v0.3.0"></a>
+            alt="docker pull infiniflow/ragflow:v0.3.1"></a>
       <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
     <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=7d09f1" alt="license">
   </a>
@@ -55,7 +58,7 @@
 
 ## 📌 新增功能
 
-- 2024-04-19 支持对话 API([更多](./docs/conversation_api.md)).
+- 2024-04-19 支持对话 API ([更多](./docs/conversation_api.md)).
 - 2024-04-16 添加嵌入模型 [BCEmbedding](https://github.com/netease-youdao/BCEmbedding) 。
 - 2024-04-16 添加 [FastEmbed](https://github.com/qdrant/fastembed) 专为轻型和高速嵌入而设计。
 - 2024-04-11 支持用 [Xinference](./docs/xinference.md) 本地化部署大模型。
@@ -74,7 +77,8 @@
 ### 📝 前提条件
 
 - CPU >= 4 核
-- RAM >= 12 GB
+- RAM >= 16 GB
+- Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
   > 如果你并没有在本机安装 Docker（Windows、Mac，或者 Linux）, 可以参考文档 [Install Docker Engine](https://docs.docker.com/engine/install/) 自行安装。
 
@@ -138,6 +142,7 @@
     * Running on http://x.x.x.x:9380
     INFO:werkzeug:Press CTRL+C to quit
    ```
+   > 如果您跳过这一步系统确认步骤就登录 RAGFlow，你的浏览器有可能会提示 `network anomaly` 或 `网络异常`，因为 RAGFlow 可能并未完全启动成功。
 
 5. 在你的浏览器中输入你的服务器对应的 IP 地址并登录 RAGFlow。
    > 上面这个例子中，您只需输入 http://IP_OF_YOUR_MACHINE 即可：未改动过配置则无需输入端口（默认的 HTTP 服务端口 80）。
@@ -174,7 +179,7 @@
 ```bash
 $ git clone https://github.com/infiniflow/ragflow.git
 $ cd ragflow/
-$ docker build -t infiniflow/ragflow:v0.3.0 .
+$ docker build -t infiniflow/ragflow:v0.3.1 .
 $ cd ragflow/docker
 $ chmod +x ./entrypoint.sh
 $ docker compose up -d

api/apps/__init__.py

@@ -54,7 +54,7 @@ app.errorhandler(Exception)(server_error_response)
 #app.config["LOGIN_DISABLED"] = True
 app.config["SESSION_PERMANENT"] = False
 app.config["SESSION_TYPE"] = "filesystem"
-app.config['MAX_CONTENT_LENGTH'] = os.environ.get("MAX_CONTENT_LENGTH", 128 * 1024 * 1024)
+app.config['MAX_CONTENT_LENGTH'] = int(os.environ.get("MAX_CONTENT_LENGTH", 128 * 1024 * 1024))
 
 Session(app)
 login_manager = LoginManager()

api/apps/api_app.py

@@ -13,18 +13,28 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import os
+import re
 from datetime import datetime, timedelta
 from flask import request
 from flask_login import login_required, current_user
+
+from api.db import FileType, ParserType
 from api.db.db_models import APIToken, API4Conversation
+from api.db.services import duplicate_name
 from api.db.services.api_service import APITokenService, API4ConversationService
 from api.db.services.dialog_service import DialogService, chat
+from api.db.services.document_service import DocumentService
+from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import UserTenantService
 from api.settings import RetCode
 from api.utils import get_uuid, current_timestamp, datetime_format
 from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request
 from itsdangerous import URLSafeTimedSerializer
 
+from api.utils.file_utils import filename_type, thumbnail
+from rag.utils import MINIO
+
 
 def generate_confirmation_token(tenent_id):
     serializer = URLSafeTimedSerializer(tenent_id)
@@ -191,4 +201,74 @@ def get(conversation_id):
 
         return get_json_result(data=conv.to_dict())
     except Exception as e:
-        return server_error_response(e)
+        return server_error_response(e)
+
+
+@manager.route('/document/upload', methods=['POST'])
+@validate_request("kb_name")
+def upload():
+    token = request.headers.get('Authorization').split()[1]
+    objs = APIToken.query(token=token)
+    if not objs:
+        return get_json_result(
+            data=False, retmsg='Token is not valid!"', retcode=RetCode.AUTHENTICATION_ERROR)
+
+    kb_name = request.form.get("kb_name").strip()
+    tenant_id = objs[0].tenant_id
+
+    try:
+        e, kb = KnowledgebaseService.get_by_name(kb_name, tenant_id)
+        if not e:
+            return get_data_error_result(
+                retmsg="Can't find this knowledgebase!")
+        kb_id = kb.id
+    except Exception as e:
+        return server_error_response(e)
+
+    if 'file' not in request.files:
+        return get_json_result(
+            data=False, retmsg='No file part!', retcode=RetCode.ARGUMENT_ERROR)
+
+    file = request.files['file']
+    if file.filename == '':
+        return get_json_result(
+            data=False, retmsg='No file selected!', retcode=RetCode.ARGUMENT_ERROR)
+    try:
+        if DocumentService.get_doc_count(kb.tenant_id) >= int(os.environ.get('MAX_FILE_NUM_PER_USER', 8192)):
+            return get_data_error_result(
+                retmsg="Exceed the maximum file number of a free user!")
+
+        filename = duplicate_name(
+            DocumentService.query,
+            name=file.filename,
+            kb_id=kb_id)
+        filetype = filename_type(filename)
+        if not filetype:
+            return get_data_error_result(
+                retmsg="This type of file has not been supported yet!")
+
+        location = filename
+        while MINIO.obj_exist(kb_id, location):
+            location += "_"
+        blob = request.files['file'].read()
+        MINIO.put(kb_id, location, blob)
+        doc = {
+            "id": get_uuid(),
+            "kb_id": kb.id,
+            "parser_id": kb.parser_id,
+            "parser_config": kb.parser_config,
+            "created_by": kb.tenant_id,
+            "type": filetype,
+            "name": filename,
+            "location": location,
+            "size": len(blob),
+            "thumbnail": thumbnail(filename, blob)
+        }
+        if doc["type"] == FileType.VISUAL:
+            doc["parser_id"] = ParserType.PICTURE.value
+        if re.search(r"\.(ppt|pptx|pages)$", filename):
+            doc["parser_id"] = ParserType.PRESENTATION.value
+        doc = DocumentService.insert(doc)
+        return get_json_result(data=doc.to_json())
+    except Exception as e:
+        return server_error_response(e)

api/apps/document_app.py

@@ -58,7 +58,8 @@ def upload():
         if not e:
             return get_data_error_result(
                 retmsg="Can't find this knowledgebase!")
-        if DocumentService.get_doc_count(kb.tenant_id) >= int(os.environ.get('MAX_FILE_NUM_PER_USER', 8192)):
+        MAX_FILE_NUM_PER_USER = int(os.environ.get('MAX_FILE_NUM_PER_USER', 0))
+        if MAX_FILE_NUM_PER_USER > 0 and DocumentService.get_doc_count(kb.tenant_id) >= MAX_FILE_NUM_PER_USER:
             return get_data_error_result(
                 retmsg="Exceed the maximum file number of a free user!")
 

api/apps/llm_app.py

@@ -28,7 +28,7 @@ from rag.llm import EmbeddingModel, ChatModel
 def factories():
     try:
         fac = LLMFactoriesService.get_all()
-        return get_json_result(data=[f.to_dict() for f in fac if f.name not in ["QAnything", "FastEmbed"]])
+        return get_json_result(data=[f.to_dict() for f in fac if f.name not in ["Youdao", "FastEmbed"]])
     except Exception as e:
         return server_error_response(e)
 
@@ -174,7 +174,7 @@ def list():
         llms = [m.to_dict()
                 for m in llms if m.status == StatusEnum.VALID.value]
         for m in llms:
-            m["available"] = m["fid"] in facts or m["llm_name"].lower() == "flag-embedding" or m["fid"] in ["QAnything","FastEmbed"]
+            m["available"] = m["fid"] in facts or m["llm_name"].lower() == "flag-embedding" or m["fid"] in ["Youdao","FastEmbed"]
 
         llm_set = set([m["llm_name"] for m in llms])
         for o in objs:

api/db/db_models.py

@@ -697,7 +697,7 @@ class Dialog(DataBaseModel):
         null=True,
         default="Chinese",
         help_text="English|Chinese")
-    llm_id = CharField(max_length=32, null=False, help_text="default llm ID")
+    llm_id = CharField(max_length=128, null=False, help_text="default llm ID")
     llm_setting = JSONField(null=False, default={"temperature": 0.1, "top_p": 0.3, "frequency_penalty": 0.7,
                                                  "presence_penalty": 0.4, "max_tokens": 215})
     prompt_type = CharField(

api/db/init_data.py

@@ -120,7 +120,7 @@ factory_infos = [{
     "tags": "LLM,TEXT EMBEDDING,SPEECH2TEXT,MODERATION",
         "status": "1",
 },{
-    "name": "QAnything",
+    "name": "Youdao",
     "logo": "",
     "tags": "LLM,TEXT EMBEDDING,SPEECH2TEXT,MODERATION",
         "status": "1",
@@ -323,7 +323,7 @@ def init_llm_factory():
             "max_tokens": 2147483648,
             "model_type": LLMType.EMBEDDING.value
         },
-        # ------------------------ QAnything -----------------------
+        # ------------------------ Youdao -----------------------
         {
             "fid": factory_infos[7]["name"],
             "llm_name": "maidalun1020/bce-embedding-base_v1",
@@ -347,7 +347,9 @@ def init_llm_factory():
     LLMService.filter_delete([LLM.fid == "Local"])
     LLMService.filter_delete([LLM.fid == "Moonshot", LLM.llm_name == "flag-embedding"])
     TenantLLMService.filter_delete([TenantLLM.llm_factory == "Moonshot", TenantLLM.llm_name == "flag-embedding"])
-
+    LLMFactoriesService.filter_update([LLMFactoriesService.model.name == "QAnything"], {"name": "Youdao"})
+    LLMService.filter_update([LLMService.model.fid == "QAnything"], {"fid": "Youdao"})
+    TenantLLMService.filter_update([TenantLLMService.model.llm_factory == "QAnything"], {"llm_factory": "Youdao"})
     """
     drop table llm;
     drop table llm_factories;

api/db/services/knowledgebase_service.py

@@ -27,7 +27,8 @@ class KnowledgebaseService(CommonService):
                           page_number, items_per_page, orderby, desc):
         kbs = cls.model.select().where(
             ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
-             TenantPermission.TEAM.value)) | (cls.model.tenant_id == user_id))
+                                                            TenantPermission.TEAM.value)) | (
+                         cls.model.tenant_id == user_id))
             & (cls.model.status == StatusEnum.VALID.value)
         )
         if desc:
@@ -56,7 +57,8 @@ class KnowledgebaseService(CommonService):
             cls.model.chunk_num,
             cls.model.parser_id,
             cls.model.parser_config]
-        kbs = cls.model.select(*fields).join(Tenant, on=((Tenant.id == cls.model.tenant_id) & (Tenant.status == StatusEnum.VALID.value))).where(
+        kbs = cls.model.select(*fields).join(Tenant, on=(
+                    (Tenant.id == cls.model.tenant_id) & (Tenant.status == StatusEnum.VALID.value))).where(
             (cls.model.id == kb_id),
             (cls.model.status == StatusEnum.VALID.value)
         )
@@ -86,6 +88,7 @@ class KnowledgebaseService(CommonService):
                     old[k] = list(set(old[k] + v))
                 else:
                     old[k] = v
+
         dfs_update(m.parser_config, config)
         cls.update_by_id(id, {"parser_config": m.parser_config})
 
@@ -97,3 +100,15 @@ class KnowledgebaseService(CommonService):
             if k.parser_config and "field_map" in k.parser_config:
                 conf.update(k.parser_config["field_map"])
         return conf
+
+    @classmethod
+    @DB.connection_context()
+    def get_by_name(cls, kb_name, tenant_id):
+        kb = cls.model.select().where(
+            (cls.model.name == kb_name)
+            & (cls.model.tenant_id == tenant_id)
+            & (cls.model.status == StatusEnum.VALID.value)
+        )
+        if kb:
+            return True, kb[0]
+        return False, None

api/db/services/llm_service.py

@@ -81,7 +81,7 @@ class TenantLLMService(CommonService):
         if not model_config:
             if llm_type == LLMType.EMBEDDING.value:
                 llm = LLMService.query(llm_name=llm_name)
-                if llm and llm[0].fid in ["QAnything", "FastEmbed"]:
+                if llm and llm[0].fid in ["Youdao", "FastEmbed"]:
                     model_config = {"llm_factory": llm[0].fid, "api_key":"", "llm_name": llm_name, "api_base": ""}
             if not model_config:
                 if llm_name == "flag-embedding":

api/db/services/task_service.py

@@ -21,6 +21,7 @@ from api.db import StatusEnum, FileType, TaskStatus
 from api.db.db_models import Task, Document, Knowledgebase, Tenant
 from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
+from api.utils import current_timestamp
 
 
 class TaskService(CommonService):
@@ -70,6 +71,25 @@ class TaskService(CommonService):
                 cls.model.id == docs[0]["id"]).execute()
             return docs
 
+    @classmethod
+    @DB.connection_context()
+    def get_ongoing_doc_name(cls):
+        with DB.lock("get_task", -1):
+            docs = cls.model.select(*[Document.kb_id, Document.location]) \
+                .join(Document, on=(cls.model.doc_id == Document.id)) \
+                .where(
+                    Document.status == StatusEnum.VALID.value,
+                    Document.run == TaskStatus.RUNNING.value,
+                    ~(Document.type == FileType.VIRTUAL.value),
+                    cls.model.progress >= 0,
+                    cls.model.progress < 1,
+                    cls.model.create_time >= current_timestamp() - 180000
+                )
+            docs = list(docs.dicts())
+            if not docs: return []
+
+            return list(set([(d["kb_id"], d["location"]) for d in docs]))
+
     @classmethod
     @DB.connection_context()
     def do_cancel(cls, id):

api/utils/file_utils.py

@@ -147,7 +147,7 @@ def filename_type(filename):
         return FileType.PDF.value
 
     if re.match(
-            r".*\.(docx|doc|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md)$", filename):
+            r".*\.(doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md)$", filename):
         return FileType.DOC.value
 
     if re.match(

conf/mapping.json

@@ -1,7 +1,7 @@
  {
   "settings": {
     "index": {
-      "number_of_shards": 4,
+      "number_of_shards": 2,
       "number_of_replicas": 0,
       "refresh_interval" : "1000ms"
     },

conf/service_conf.yaml

@@ -13,11 +13,16 @@ minio:
   user: 'rag_flow'
   password: 'infini_rag_flow'
   host: 'minio:9000'
+redis:
+  db: 1
+  password: 'infini_rag_flow'
+  host: 'redis:6379'
 es:
   hosts: 'http://es01:9200'
 user_default_llm:
   factory: 'Tongyi-Qianwen'
   api_key: 'sk-xxxxxxxxxxxxx'
+  base_url: ''
 oauth:
   github:
     client_id: xxxxxxxxxxxxxxxxxxxxxxxxx

deepdoc/README_zh.md

@@ -1 +1,116 @@
-[English](./README.md) | 简体中文
+[English](./README.md) | 简体中文
+
+# *Deep*Doc
+
+- [*Deep*Doc](#deepdoc)
+  - [1. 介绍](#1-介绍)
+  - [2. 视觉处理](#2-视觉处理)
+  - [3. 解析器](#3-解析器)
+    - [简历](#简历)
+
+<a name="1"></a>
+## 1. 介绍
+
+对于来自不同领域、具有不同格式和不同检索要求的大量文档，准确的分析成为一项极具挑战性的任务。*Deep*Doc 就是为了这个目的而诞生的。到目前为止，*Deep*Doc 中有两个组成部分：视觉处理和解析器。如果您对我们的OCR、布局识别和TSR结果感兴趣，您可以运行下面的测试程序。
+
+```bash
+python deepdoc/vision/t_ocr.py -h
+usage: t_ocr.py [-h] --inputs INPUTS [--output_dir OUTPUT_DIR]
+
+options:
+  -h, --help            show this help message and exit
+  --inputs INPUTS       Directory where to store images or PDFs, or a file path to a single image or PDF
+  --output_dir OUTPUT_DIR
+                        Directory where to store the output images. Default: './ocr_outputs'
+```
+
+```bash
+python deepdoc/vision/t_recognizer.py -h
+usage: t_recognizer.py [-h] --inputs INPUTS [--output_dir OUTPUT_DIR] [--threshold THRESHOLD] [--mode {layout,tsr}]
+
+options:
+  -h, --help            show this help message and exit
+  --inputs INPUTS       Directory where to store images or PDFs, or a file path to a single image or PDF
+  --output_dir OUTPUT_DIR
+                        Directory where to store the output images. Default: './layouts_outputs'
+  --threshold THRESHOLD
+                        A threshold to filter out detections. Default: 0.5
+  --mode {layout,tsr}   Task mode: layout recognition or table structure recognition
+```
+
+HuggingFace为我们的模型提供服务。如果你在下载HuggingFace模型时遇到问题，这可能会有所帮助！！
+
+```bash
+export HF_ENDPOINT=https://hf-mirror.com
+```
+
+<a name="2"></a>
+## 2. 视觉处理
+
+作为人类，我们使用视觉信息来解决问题。
+
+  - **OCR（Optical Character Recognition，光学字符识别）**。由于许多文档都是以图像形式呈现的，或者至少能够转换为图像，因此OCR是文本提取的一个非常重要、基本，甚至通用的解决方案。
+
+    ```bash
+    python deepdoc/vision/t_ocr.py --inputs=path_to_images_or_pdfs --output_dir=path_to_store_result
+    ```
+
+    输入可以是图像或PDF的目录，或者单个图像、PDF文件。您可以查看文件夹 `path_to_store_result` ，其中有演示结果位置的图像，以及包含OCR文本的txt文件。
+    
+    <div align="center" style="margin-top:20px;margin-bottom:20px;">
+    <img src="https://github.com/infiniflow/ragflow/assets/12318111/f25bee3d-aaf7-4102-baf5-d5208361d110" width="900"/>
+    </div>
+
+  - 布局识别（Layout recognition）。来自不同领域的文件可能有不同的布局，如报纸、杂志、书籍和简历在布局方面是不同的。只有当机器有准确的布局分析时，它才能决定这些文本部分是连续的还是不连续的，或者这个部分需要表结构识别（Table Structure Recognition，TSR）来处理，或者这个部件是一个图形并用这个标题来描述。我们有10个基本布局组件，涵盖了大多数情况：
+      - 文本
+      - 标题
+      - 配图
+      - 配图标题
+      - 表格
+      - 表格标题
+      - 页头
+      - 页尾
+      - 参考引用
+      - 公式
+      
+     请尝试以下命令以查看布局检测结果。
+
+    ```bash
+    python deepdoc/vision/t_recognizer.py --inputs=path_to_images_or_pdfs --threshold=0.2 --mode=layout --output_dir=path_to_store_result
+    ```
+
+    输入可以是图像或PDF的目录，或者单个图像、PDF文件。您可以查看文件夹 `path_to_store_result` ，其中有显示检测结果的图像，如下所示：
+    <div align="center" style="margin-top:20px;margin-bottom:20px;">
+    <img src="https://github.com/infiniflow/ragflow/assets/12318111/07e0f625-9b28-43d0-9fbb-5bf586cd286f" width="1000"/>
+    </div>
+  
+  - **TSR（Table Structure Recognition，表结构识别）**。数据表是一种常用的结构，用于表示包括数字或文本在内的数据。表的结构可能非常复杂，比如层次结构标题、跨单元格和投影行标题。除了TSR，我们还将内容重新组合成LLM可以很好理解的句子。TSR任务有五个标签：
+      - 列
+      - 行
+      - 列标题
+      - 行标题
+      - 合并单元格
+      
+    请尝试以下命令以查看布局检测结果。
+
+    ```bash
+    python deepdoc/vision/t_recognizer.py --inputs=path_to_images_or_pdfs --threshold=0.2 --mode=tsr --output_dir=path_to_store_result
+    ```
+
+    输入可以是图像或PDF的目录，或者单个图像、PDF文件。您可以查看文件夹 `path_to_store_result` ，其中包含图像和html页面，这些页面展示了以下检测结果：
+
+    <div align="center" style="margin-top:20px;margin-bottom:20px;">
+    <img src="https://github.com/infiniflow/ragflow/assets/12318111/cb24e81b-f2ba-49f3-ac09-883d75606f4c" width="1000"/>
+    </div>
+        
+<a name="3"></a>
+## 3. 解析器
+
+PDF、DOCX、EXCEL和PPT四种文档格式都有相应的解析器。最复杂的是PDF解析器，因为PDF具有灵活性。PDF解析器的输出包括：
+  - 在PDF中有自己位置的文本块（页码和矩形位置）。
+  - 带有PDF裁剪图像的表格，以及已经翻译成自然语言句子的内容。
+  - 图中带标题和文字的图。
+  
+### 简历
+
+简历是一种非常复杂的文件。一份由各种布局的非结构化文本组成的简历可以分解为由近百个字段组成的结构化数据。我们还没有打开解析器，因为我们在解析过程之后打开了处理方法。

deepdoc/parser/pdf_parser.py

@@ -11,7 +11,7 @@ import pdfplumber
 import logging
 from PIL import Image, ImageDraw
 import numpy as np
-
+from timeit import default_timer as timer
 from PyPDF2 import PdfReader as pdf2_read
 
 from api.utils.file_utils import get_project_base_directory
@@ -37,8 +37,8 @@ class HuParser:
             self.updown_cnt_mdl.set_param({"device": "cuda"})
         try:
             model_dir = os.path.join(
-                    get_project_base_directory(),
+                get_project_base_directory(),
-                    "rag/res/deepdoc")
+                "rag/res/deepdoc")
             self.updown_cnt_mdl.load_model(os.path.join(
                 model_dir, "updown_concat_xgb.model"))
         except Exception as e:
@@ -49,7 +49,6 @@ class HuParser:
             self.updown_cnt_mdl.load_model(os.path.join(
                 model_dir, "updown_concat_xgb.model"))
 
-
         self.page_from = 0
         """
         If you have trouble downloading HuggingFace models, -_^ this might help!!
@@ -76,7 +75,7 @@ class HuParser:
     def _y_dis(
             self, a, b):
         return (
-            b["top"] + b["bottom"] - a["top"] - a["bottom"]) / 2
+                       b["top"] + b["bottom"] - a["top"] - a["bottom"]) / 2
 
     def _match_proj(self, b):
         proj_patt = [
@@ -99,9 +98,9 @@ class HuParser:
         tks_down = huqie.qie(down["text"][:LEN]).split(" ")
         tks_up = huqie.qie(up["text"][-LEN:]).split(" ")
         tks_all = up["text"][-LEN:].strip() \
-            + (" " if re.match(r"[a-zA-Z0-9]+",
+                  + (" " if re.match(r"[a-zA-Z0-9]+",
-                               up["text"][-1] + down["text"][0]) else "") \
+                                     up["text"][-1] + down["text"][0]) else "") \
-            + down["text"][:LEN].strip()
+                  + down["text"][:LEN].strip()
         tks_all = huqie.qie(tks_all).split(" ")
         fea = [
             up.get("R", -1) == down.get("R", -1),
@@ -123,7 +122,7 @@ class HuParser:
             True if re.search(r"[，,][^。.]+$", up["text"]) else False,
             True if re.search(r"[，,][^。.]+$", up["text"]) else False,
             True if re.search(r"[\(（][^\)）]+$", up["text"])
-            and re.search(r"[\)）]", down["text"]) else False,
+                    and re.search(r"[\)）]", down["text"]) else False,
             self._match_proj(down),
             True if re.match(r"[A-Z]", down["text"]) else False,
             True if re.match(r"[A-Z]", up["text"][-1]) else False,
@@ -185,7 +184,7 @@ class HuParser:
                 continue
             for tb in tbls:  # for table
                 left, top, right, bott = tb["x0"] - MARGIN, tb["top"] - MARGIN, \
-                    tb["x1"] + MARGIN, tb["bottom"] + MARGIN
+                                         tb["x1"] + MARGIN, tb["bottom"] + MARGIN
                 left *= ZM
                 top *= ZM
                 right *= ZM
@@ -297,7 +296,7 @@ class HuParser:
         for b in bxs:
             if not b["text"]:
                 left, right, top, bott = b["x0"] * ZM, b["x1"] * \
-                    ZM, b["top"] * ZM, b["bottom"] * ZM
+                                         ZM, b["top"] * ZM, b["bottom"] * ZM
                 b["text"] = self.ocr.recognize(np.array(img),
                                                np.array([[left, top], [right, top], [right, bott], [left, bott]],
                                                         dtype=np.float32))
@@ -622,7 +621,7 @@ class HuParser:
                 i += 1
                 continue
             lout_no = str(self.boxes[i]["page_number"]) + \
-                "-" + str(self.boxes[i]["layoutno"])
+                      "-" + str(self.boxes[i]["layoutno"])
             if TableStructureRecognizer.is_caption(self.boxes[i]) or self.boxes[i]["layout_type"] in ["table caption",
                                                                                                       "title",
                                                                                                       "figure caption",
@@ -936,6 +935,7 @@ class HuParser:
         self.page_cum_height = [0]
         self.page_layout = []
         self.page_from = page_from
+        st = timer()
         try:
             self.pdf = pdfplumber.open(fnm) if isinstance(
                 fnm, str) else pdfplumber.open(BytesIO(fnm))
@@ -974,6 +974,7 @@ class HuParser:
                         self.outlines.append((a["/Title"], depth))
                         continue
                     dfs(a, depth + 1)
+
             dfs(outlines, 0)
         except Exception as e:
             logging.warning(f"Outlines exception: {e}")
@@ -983,13 +984,15 @@ class HuParser:
         logging.info("Images converted.")
         self.is_english = [re.search(r"[a-zA-Z0-9,/¸;:'\[\]\(\)!@#$%^&*\"?<>._-]{30,}", "".join(
             random.choices([c["text"] for c in self.page_chars[i]], k=min(100, len(self.page_chars[i]))))) for i in
-            range(len(self.page_chars))]
+                           range(len(self.page_chars))]
         if sum([1 if e else 0 for e in self.is_english]) > len(
                 self.page_images) / 2:
             self.is_english = True
         else:
             self.is_english = False
+        self.is_english = False
 
+        st = timer()
         for i, img in enumerate(self.page_images):
             chars = self.page_chars[i] if not self.is_english else []
             self.mean_height.append(
@@ -1007,15 +1010,11 @@ class HuParser:
                                                                        chars[j]["width"]) / 2:
                     chars[j]["text"] += " "
                 j += 1
-            # if i > 0:
+
-            #     if not chars:
-            #         self.page_cum_height.append(img.size[1] / zoomin)
-            #     else:
-            #         self.page_cum_height.append(
-            #             np.max([c["bottom"] for c in chars]))
             self.__ocr(i + 1, img, chars, zoomin)
-            if callback:
+            if callback and i % 6 == 5:
                 callback(prog=(i + 1) * 0.6 / len(self.page_images), msg="")
+        # print("OCR:", timer()-st)
 
         if not self.is_english and not any(
                 [c for c in self.page_chars]) and self.boxes:
@@ -1051,7 +1050,7 @@ class HuParser:
             left, right, top, bottom = float(left), float(
                 right), float(top), float(bottom)
             poss.append(([int(p) - 1 for p in pn.split("-")],
-                        left, right, top, bottom))
+                         left, right, top, bottom))
         if not poss:
             if need_position:
                 return None, None
@@ -1077,7 +1076,7 @@ class HuParser:
                 self.page_images[pns[0]].crop((left * ZM, top * ZM,
                                                right *
                                                ZM, min(
-                                                   bottom, self.page_images[pns[0]].size[1])
+                    bottom, self.page_images[pns[0]].size[1])
                                                ))
             )
             if 0 < ii < len(poss) - 1:

docker/.env

@@ -11,7 +11,9 @@ ES_PORT=1200
 KIBANA_PORT=6601
 
 # Increase or decrease based on the available host memory (in bytes)
-MEM_LIMIT=4073741824
+
+MEM_LIMIT=8073741824
+
 
 MYSQL_PASSWORD=infini_rag_flow
 MYSQL_PORT=5455
@@ -25,7 +27,7 @@ MINIO_PASSWORD=infini_rag_flow
 
 SVR_HTTP_PORT=9380
 
-RAGFLOW_VERSION=v0.3.0
+RAGFLOW_VERSION=v0.3.1
 
 TIMEZONE='Asia/Shanghai'
 

docker/docker-compose-CN-oc9.yml

@@ -0,0 +1,29 @@
+include:
+  - path: ./docker-compose-base.yml
+    env_file: ./.env
+
+services:
+   ragflow:
+    depends_on:
+      mysql:
+        condition: service_healthy
+      es01:
+        condition: service_healthy
+    image: edwardelric233/ragflow:oc9
+    container_name: ragflow-server
+    ports:
+      - ${SVR_HTTP_PORT}:9380
+      - 80:80
+      - 443:443
+    volumes:
+      - ./service_conf.yaml:/ragflow/conf/service_conf.yaml
+      - ./ragflow-logs:/ragflow/logs
+      - ./nginx/ragflow.conf:/etc/nginx/conf.d/ragflow.conf
+      - ./nginx/proxy.conf:/etc/nginx/proxy.conf
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
+    environment:
+      - TZ=${TIMEZONE}
+      - HF_ENDPOINT=https://hf-mirror.com
+    networks:
+      - ragflow
+    restart: always

docker/docker-compose-base.yml

@@ -29,23 +29,23 @@ services:
       - ragflow
     restart: always
 
-  kibana:
+  #kibana:
-    depends_on:
+  #  depends_on:
-        es01:
+  #      es01:
-          condition: service_healthy
+  #        condition: service_healthy
-    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
+  #  image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
-    container_name: ragflow-kibana
+  #  container_name: ragflow-kibana
-    volumes:
+  #  volumes:
-      - kibanadata:/usr/share/kibana/data
+  #    - kibanadata:/usr/share/kibana/data
-    ports:
+  #  ports:
-      - ${KIBANA_PORT}:5601
+  #    - ${KIBANA_PORT}:5601
-    environment:
+  #  environment:
-      - SERVERNAME=kibana
+  #    - SERVERNAME=kibana
-      - ELASTICSEARCH_HOSTS=http://es01:9200
+  #    - ELASTICSEARCH_HOSTS=http://es01:9200
-      - TZ=${TIMEZONE}
+  #    - TZ=${TIMEZONE}
-    mem_limit: ${MEM_LIMIT}
+  #  mem_limit: ${MEM_LIMIT}
-    networks:
+  #  networks:
-      - ragflow
+  #    - ragflow
 
   mysql:
     image: mysql:5.7.18

docker/docker-compose.yml

@@ -12,7 +12,6 @@ services:
     image: infiniflow/ragflow:${RAGFLOW_VERSION}
     container_name: ragflow-server
     ports:
-      - ${SVR_HTTP_PORT}:9380
       - ${SVR_HTTP_PORT}:9380
       - 80:80
       - 443:443

docker/entrypoint.sh

@@ -23,13 +23,12 @@ function watch_broker(){
 }
 
 function task_bro(){
-    sleep 160;
     watch_broker;
 }
 
 task_bro &
 
-WS=2
+WS=1
 for ((i=0;i<WS;i++))
 do
   task_exe $i $WS &

docker/service_conf.yaml

@@ -13,6 +13,10 @@ minio:
   user: 'rag_flow'
   password: 'infini_rag_flow'
   host: 'minio:9000'
+redis:
+  db: 1
+  password: 'infini_rag_flow'
+  host: 'redis:6379'
 es:
   hosts: 'http://es01:9200'
 user_default_llm:

docs/conversation_api.md

@@ -11,7 +11,7 @@ https://demo.ragflow.io/v1/
 
 ## Authorization
 
-All the APIs are authorized with API-Key. Please keep it save and private. Don't reveal it in any way from the front-end. 
+All the APIs are authorized with API-Key. Please keep it safe and private. Don't reveal it in any way from the front-end. 
 The API-Key should put in the header of request:
 ```buildoutcfg
 Authorization: Bearer {API_KEY}
@@ -303,5 +303,61 @@ This will be called to get the answer to users' questions.
 ## Get document content or image
 
 This is usually used when display content of citation.
-### Path: /document/get/\<id\>
+### Path: /api/document/get/\<id\>
 ### Method: GET
+
+## Upload file
+
+This is usually used when upload a file to.
+### Path: /api/document/upload/
+### Method: POST
+
+### Parameter:
+
+| name    | type   | optional | description                            |
+|---------|--------|----------|----------------------------------------|
+| file    | file   | No       | Upload file.                           |
+| kb_name | string | No       | Choose the upload knowledge base name. |
+
+### Response 
+```json
+{
+    "data": {
+        "chunk_num": 0,
+        "create_date": "Thu, 25 Apr 2024 14:30:06 GMT",
+        "create_time": 1714026606921,
+        "created_by": "553ec818fd5711ee8ea63043d7ed348e",
+        "id": "41e9324602cd11ef9f5f3043d7ed348e",
+        "kb_id": "06802686c0a311ee85d6246e9694c130",
+        "location": "readme.txt",
+        "name": "readme.txt",
+        "parser_config": {
+            "field_map": {
+            },
+            "pages": [
+                [
+                    0,
+                    1000000
+                ]
+            ]
+        },
+        "parser_id": "general",
+        "process_begin_at": null,
+        "process_duation": 0.0,
+        "progress": 0.0,
+        "progress_msg": "",
+        "run": "0",
+        "size": 929,
+        "source_type": "local",
+        "status": "1",
+        "thumbnail": null,
+        "token_num": 0,
+        "type": "doc",
+        "update_date": "Thu, 25 Apr 2024 14:30:06 GMT",
+        "update_time": 1714026606921
+    },
+    "retcode": 0,
+    "retmsg": "success"
+}
+
+```

docs/faq.md

@@ -2,112 +2,108 @@
 
 ## General
 
-### What sets RAGFlow apart from other RAG products?
+### 1. What sets RAGFlow apart from other RAG products?
 
 The "garbage in garbage out" status quo remains unchanged despite the fact that LLMs have advanced Natural Language Processing (NLP) significantly. In response, RAGFlow introduces two unique features compared to other Retrieval-Augmented Generation (RAG) products.
 
 - Fine-grained document parsing: Document parsing involves images and tables, with the flexibility for you to intervene as needed.
 - Traceable answers with reduced hallucinations: You can trust RAGFlow's responses as you can view the citations and references supporting them.
 
-### Which languages does RAGFlow support?
+### 2. Which languages does RAGFlow support?
 
 English, simplified Chinese, traditional Chinese for now. 
 
 ## Performance
 
-### Why does it take longer for RAGFlow to parse a document than LangChain?
+### 1. Why does it take longer for RAGFlow to parse a document than LangChain?
 
 We put painstaking effort into document pre-processing tasks like layout analysis, table structure recognition, and OCR (Optical Character Recognition) using our vision model. This contributes to the additional time required. 
 
+### 2. Why does RAGFlow require more resources than other projects?
+
+RAGFlow has a number of built-in models for document structure parsing, which account for the additional computational resources.
+
 ## Feature
 
-### Which architectures or devices does RAGFlow support?
+### 1. Which architectures or devices does RAGFlow support?
 
-ARM64 and Ascend GPU are not supported.
+Currently, we only support x86 CPU and Nvidia GPU. 
 
-### Do you offer an API for integration with third-party applications?
+### 2. Do you offer an API for integration with third-party applications?
 
-These APIs are still in development. Contributions are welcome.
+The corresponding APIs are now available. See the [Conversation API](./conversation_api.md) for more information. 
 
-### Do you support stream output?
+### 3. Do you support stream output?
 
 No, this feature is still in development. Contributions are welcome. 
 
-### Is it possible to share dialogue through URL?
+### 4. Is it possible to share dialogue through URL?
+
+Yes, this feature is now available.
+
+### 5. Do you support multiple rounds of dialogues, i.e., referencing previous dialogues as context for the current dialogue?
 
 This feature and the related APIs are still in development. Contributions are welcome.
 
-### Do you support multiple rounds of dialogues, i.e., referencing previous dialogues as context for the current dialogue?
 
-This feature and the related APIs are still in development. Contributions are welcome.
+## Troubleshooting
 
-## Configurations
+### 1. Issues with docker images
 
-### How to increase the length of RAGFlow responses?
+#### 1.1 How to build the RAGFlow image from scratch?
 
-1. Right click the desired dialog to display the **Chat Configuration** window.
-2. Switch to the **Model Setting** tab and adjust the **Max Tokens** slider to get the desired length.
-3. Click **OK** to confirm your change. 
-
-
-### What does Empty response mean? How to set it?
-
-You limit what the system responds to what you specify in **Empty response** if nothing is retrieved from your knowledge base. If you do not specify anything in **Empty response**, you let your LLM improvise, giving it a chance to hallucinate.
-
-### Can I set the base URL for OpenAI somewhere?
-
-![](https://github.com/infiniflow/ragflow/assets/93570324/8cfb6fa4-8a97-415d-b9fa-b6f405a055f3)
-
-
-### How to run RAGFlow with a locally deployed LLM?
-
-You can use Ollama to deploy local LLM. See [here](https://github.com/infiniflow/ragflow/blob/main/docs/ollama.md) for more information. 
-
-### How to link up ragflow and ollama servers?
-
-- If RAGFlow is locally deployed, ensure that your RAGFlow and Ollama are in the same LAN. 
-- If you are using our online demo, ensure that the IP address of your Ollama server is public and accessible.
-
-### How to configure RAGFlow to respond with 100% matched results, rather than utilizing LLM?
-
-1. Click the **Knowledge Base** tab in the middle top of the page.
-2. Right click the desired knowledge base to display the **Configuration** dialogue. 
-3. Choose **Q&A** as the chunk method and click **Save** to confirm your change. 
-
-## Debugging
-
-### `WARNING: can't find /raglof/rag/res/borker.tm`
-
-Ignore this warning and continue. All system warnings can be ignored.
-
-### `dependency failed to start: container ragflow-mysql is unhealthy`
-
-`dependency failed to start: container ragflow-mysql is unhealthy` means that your MySQL container failed to start. If you are using a Mac with an M1/M2 chip, replace `mysql:5.7.18` with `mariadb:10.5.8` in **docker-compose-base.yml**.
-
-### `Realtime synonym is disabled, since no redis connection`
-
-Ignore this warning and continue. All system warnings can be ignored.
-
-![](https://github.com/infiniflow/ragflow/assets/93570324/ef5a6194-084a-4fe3-bdd5-1c025b40865c)
-
-### Why does it take so long to parse a 2MB document?
-
-Parsing requests have to wait in queue due to limited server resources. We are currently enhancing our algorithms and increasing computing power.
-
-### Why does my document parsing stall at under one percent?
-
-![stall](https://github.com/infiniflow/ragflow/assets/93570324/3589cc25-c733-47d5-bbfc-fedb74a3da50)
-
-If your RAGFlow is deployed *locally*, try the following: 
-
-1. Check the log of your RAGFlow server to see if it is running properly:
-```bash
-docker logs -f ragflow-server
 ```
-2. Check if the **tast_executor.py** process exist.
+$ git clone https://github.com/infiniflow/ragflow.git
-3. Check if your RAGFlow server can access hf-mirror.com or huggingface.com.
+$ cd ragflow
+$ docker build -t infiniflow/ragflow:v0.3.1 .
+$ cd ragflow/docker
+$ chmod +x ./entrypoint.sh
+$ docker compose up -d
+```
 
-### `MaxRetryError: HTTPSConnectionPool(host='hf-mirror.com', port=443)`
+#### 1.2 `process "/bin/sh -c cd ./web && npm i && npm run build"` failed
+
+1. Check your network from within Docker, for example:
+```bash
+curl https://hf-mirror.com
+```
+
+2. If your network works fine, the issue lies with the Docker network configuration. Replace the Docker building command:
+```bash
+docker build -t infiniflow/ragflow:vX.Y.Z.
+```
+   With this:  
+```bash
+docker build -t infiniflow/ragflow:vX.Y.Z. --network host
+```
+
+### 2. Issues with huggingface models
+
+#### 2.1 Cannot access https://huggingface.co
+ 
+A *locally* deployed RAGflow downloads OCR and embedding modules from [Huggingface website](https://huggingface.co) by default. If your machine is unable to access this site, the following error occurs and PDF parsing fails: 
+
+```
+FileNotFoundError: [Errno 2] No such file or directory: '/root/.cache/huggingface/hub/models--InfiniFlow--deepdoc/snapshots/be0c1e50eef6047b412d1800aa89aba4d275f997/ocr.res'
+```
+ To fix this issue, use https://hf-mirror.com instead:
+
+ 1. Stop all containers and remove all related resources:
+
+ ```bash
+ cd ragflow/docker/
+ docker compose down
+ ```
+
+ 2. Replace `https://huggingface.co` with `https://hf-mirror.com` in **ragflow/docker/docker-compose.yml**.
+ 
+ 3. Start up the server: 
+
+ ```bash
+ docker compose up -d 
+ ```
+
+#### 2.2. `MaxRetryError: HTTPSConnectionPool(host='hf-mirror.com', port=443)`
 
 This error suggests that you do not have Internet access or are unable to connect to hf-mirror.com. Try the following: 
 
@@ -117,17 +113,98 @@ This error suggests that you do not have Internet access or are unable to connec
 - ~/deepdoc:/ragflow/rag/res/deepdoc
 ```
 
-### `Index failure`
+#### 2.3 `FileNotFoundError: [Errno 2] No such file or directory: '/root/.cache/huggingface/hub/models--InfiniFlow--deepdoc/snapshots/FileNotFoundError: [Errno 2] No such file or directory: '/ragflow/rag/res/deepdoc/ocr.res'be0c1e50eef6047b412d1800aa89aba4d275f997/ocr.res'`
+
+1. Check your network from within Docker, for example: 
+```bash
+curl https://hf-mirror.com
+```
+2. Run `ifconfig` to check the `mtu` value. If the server's `mtu` is `1450` while the NIC's `mtu` in the container is `1500`, this mismatch may cause network instability. Adjust the `mtu` policy as follows:
+
+```
+vim docker-compose-base.yml
+# Original configuration：
+networks:
+  ragflow:
+    driver: bridge
+# Modified configuration：
+networks:
+  ragflow:
+    driver: bridge
+    driver_opts:
+      com.docker.network.driver.mtu: 1450
+```
+
+### 3. Issues with RAGFlow servers
+
+#### 3.1 `WARNING: can't find /raglof/rag/res/borker.tm`
+
+Ignore this warning and continue. All system warnings can be ignored.
+
+#### 3.2 `network anomaly There is an abnormality in your network and you cannot connect to the server.`
+
+![anomaly](https://github.com/infiniflow/ragflow/assets/93570324/beb7ad10-92e4-4a58-8886-bfb7cbd09e5d)
+
+You will not log in to RAGFlow unless the server is fully initialized. Run `docker logs -f ragflow-server`.
+
+*The server is successfully initialized, if your system displays the following:*
+
+```
+    ____                 ______ __
+   / __ \ ____ _ ____ _ / ____// /____  _      __
+  / /_/ // __ `// __ `// /_   / // __ \| | /| / /
+ / _, _// /_/ // /_/ // __/  / // /_/ /| |/ |/ /
+/_/ |_| \__,_/ \__, //_/    /_/ \____/ |__/|__/
+              /____/
+
+ * Running on all addresses (0.0.0.0)
+ * Running on http://127.0.0.1:9380
+ * Running on http://x.x.x.x:9380
+ INFO:werkzeug:Press CTRL+C to quit
+```
+
+
+### 4. Issues with RAGFlow backend services
+
+#### 4.1 `dependency failed to start: container ragflow-mysql is unhealthy`
+
+`dependency failed to start: container ragflow-mysql is unhealthy` means that your MySQL container failed to start. Try replacing `mysql:5.7.18` with `mariadb:10.5.8` in **docker-compose-base.yml**.
+
+#### 4.2 `Realtime synonym is disabled, since no redis connection`
+
+Ignore this warning and continue. All system warnings can be ignored.
+
+![](https://github.com/infiniflow/ragflow/assets/93570324/ef5a6194-084a-4fe3-bdd5-1c025b40865c)
+
+#### 4.3 Why does it take so long to parse a 2MB document?
+
+Parsing requests have to wait in queue due to limited server resources. We are currently enhancing our algorithms and increasing computing power.
+
+#### 4.4 Why does my document parsing stall at under one percent?
+
+![stall](https://github.com/infiniflow/ragflow/assets/93570324/3589cc25-c733-47d5-bbfc-fedb74a3da50)
+
+If your RAGFlow is deployed *locally*, try the following: 
+
+1. Check the log of your RAGFlow server to see if it is running properly:
+```bash
+docker logs -f ragflow-server
+```
+2. Check if the **task_executor.py** process exists.
+3. Check if your RAGFlow server can access hf-mirror.com or huggingface.com.
+
+
+#### 4.5 `Index failure`
 
 An index failure usually indicates an unavailable Elasticsearch service.
 
-### How to check the log of RAGFlow?
+#### 4.6 How to check the log of RAGFlow?
 
 ```bash
 tail -f path_to_ragflow/docker/ragflow-logs/rag/*.log
 ```
 
-### How to check the status of each component in RAGFlow?
+#### 4.7 How to check the status of each component in RAGFlow?
 
 ```bash
 $ docker ps
@@ -135,13 +212,13 @@ $ docker ps
 *The system displays the following if all your RAGFlow components are running properly:* 
 
 ```
-5bc45806b680   infiniflow/ragflow:v0.3.0     "./entrypoint.sh"        11 hours ago   Up 11 hours               0.0.0.0:80->80/tcp, :::80->80/tcp, 0.0.0.0:443->443/tcp, :::443->443/tcp, 0.0.0.0:9380->9380/tcp, :::9380->9380/tcp   ragflow-server
+5bc45806b680   infiniflow/ragflow:v0.3.1     "./entrypoint.sh"        11 hours ago   Up 11 hours               0.0.0.0:80->80/tcp, :::80->80/tcp, 0.0.0.0:443->443/tcp, :::443->443/tcp, 0.0.0.0:9380->9380/tcp, :::9380->9380/tcp   ragflow-server
 91220e3285dd   docker.elastic.co/elasticsearch/elasticsearch:8.11.3   "/bin/tini -- /usr/l…"   11 hours ago   Up 11 hours (healthy)     9300/tcp, 0.0.0.0:9200->9200/tcp, :::9200->9200/tcp           ragflow-es-01
 d8c86f06c56b   mysql:5.7.18        "docker-entrypoint.s…"   7 days ago     Up 16 seconds (healthy)   0.0.0.0:3306->3306/tcp, :::3306->3306/tcp     ragflow-mysql
 cd29bcb254bc   quay.io/minio/minio:RELEASE.2023-12-20T01-00-02Z       "/usr/bin/docker-ent…"   2 weeks ago    Up 11 hours      0.0.0.0:9001->9001/tcp, :::9001->9001/tcp, 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp     ragflow-minio
 ```
 
-### `Exception: Can't connect to ES cluster`
+#### 4.8 `Exception: Can't connect to ES cluster`
 
 1. Check the status of your Elasticsearch component:
 
@@ -153,7 +230,7 @@ $ docker ps
 91220e3285dd   docker.elastic.co/elasticsearch/elasticsearch:8.11.3   "/bin/tini -- /usr/l…"   11 hours ago   Up 11 hours (healthy)     9300/tcp, 0.0.0.0:9200->9200/tcp, :::9200->9200/tcp           ragflow-es-01
 ```
 
-2. If your container keeps restarting, ensure `vm.max_map_count` >= 262144 as per [this README](https://github.com/infiniflow/ragflow?tab=readme-ov-file#-start-up-the-server).
+2. If your container keeps restarting, ensure `vm.max_map_count` >= 262144 as per [this README](https://github.com/infiniflow/ragflow?tab=readme-ov-file#-start-up-the-server). Updating the `vm.max_map_count` value in **/etc/sysctl.conf** is required, if you wish to keep your change permanent. This configuration works only for Linux.
 
 
 3. If your issue persists, ensure that the ES host setting is correct:
@@ -169,22 +246,22 @@ $ docker ps
     ```
 
 
-### `{"data":null,"retcode":100,"retmsg":"<NotFound '404: Not Found'>"}`
+#### 4.9 `{"data":null,"retcode":100,"retmsg":"<NotFound '404: Not Found'>"}`
 
-Your IP address or port number may be incorrect. If you are using the default configurations, enter http://<IP_OF_YOUR_MACHINE> (**NOT `localhost`, NOT 9380, AND NO PORT NUMBER REQUIRED!**) in your browser. This should work.
+Your IP address or port number may be incorrect. If you are using the default configurations, enter http://<IP_OF_YOUR_MACHINE> (**NOT 9380, AND NO PORT NUMBER REQUIRED!**) in your browser. This should work.
 
-### `Ollama - Mistral instance running at 127.0.0.1:11434 but cannot add Ollama as model in RagFlow`
+#### 4.10 `Ollama - Mistral instance running at 127.0.0.1:11434 but cannot add Ollama as model in RagFlow`
 
 A correct Ollama IP address and port is crucial to adding models to Ollama:
 
 - If you are on demo.ragflow.io, ensure that the server hosting Ollama has a publicly accessible IP address.Note that 127.0.0.1 is not a publicly accessible IP address.
 - If you deploy RAGFlow locally, ensure that Ollama and RAGFlow are in the same LAN and can comunicate with each other.
 
-### Do you offer examples of using deepdoc to parse PDF or other files?
+#### 4.11 Do you offer examples of using deepdoc to parse PDF or other files?
 
 Yes, we do. See the Python files under the **rag/app** folder. 
 
-### Why did I fail to upload a 10MB+ file to my locally deployed RAGFlow?
+#### 4.12 Why did I fail to upload a 10MB+ file to my locally deployed RAGFlow?
 
 You probably forgot to update the **MAX_CONTENT_LENGTH** environment variable:
 
@@ -196,14 +273,14 @@ MAX_CONTENT_LENGTH=100000000
 ```
 environment:
   - MAX_CONTENT_LENGTH=${MAX_CONTENT_LENGTH}
-``` 
+```
 3. Restart the RAGFlow server:
 ```
 docker compose up ragflow -d
 ```
    *Now you should be able to upload files of sizes less than 100MB.*
 
-### `Table 'rag_flow.document' doesn't exist`
+#### 4.13 `Table 'rag_flow.document' doesn't exist`
 
 This exception occurs when starting up the RAGFlow server. Try the following: 
 
@@ -226,10 +303,47 @@ This exception occurs when starting up the RAGFlow server. Try the following:
   docker compose up
   ```
 
-### `hint : 102  Fail to access model  Connection error`
+#### 4.14 `hint : 102  Fail to access model  Connection error`
 
 ![hint102](https://github.com/infiniflow/ragflow/assets/93570324/6633d892-b4f8-49b5-9a0a-37a0a8fba3d2)
 
 1. Ensure that the RAGFlow server can access the base URL.
 2. Do not forget to append **/v1/** to **http://IP:port**: 
-   **http://IP:port/v1/**
+   **http://IP:port/v1/**
+
+
+## Usage
+
+### 1. How to increase the length of RAGFlow responses?
+
+1. Right click the desired dialog to display the **Chat Configuration** window.
+2. Switch to the **Model Setting** tab and adjust the **Max Tokens** slider to get the desired length.
+3. Click **OK** to confirm your change. 
+
+
+### 2. What does Empty response mean? How to set it?
+
+You limit what the system responds to what you specify in **Empty response** if nothing is retrieved from your knowledge base. If you do not specify anything in **Empty response**, you let your LLM improvise, giving it a chance to hallucinate.
+
+### 3. Can I set the base URL for OpenAI somewhere?
+
+![](https://github.com/infiniflow/ragflow/assets/93570324/8cfb6fa4-8a97-415d-b9fa-b6f405a055f3)
+
+### 4. How to run RAGFlow with a locally deployed LLM?
+
+You can use Ollama to deploy local LLM. See [here](https://github.com/infiniflow/ragflow/blob/main/docs/ollama.md) for more information. 
+
+### 5. How to link up ragflow and ollama servers?
+
+- If RAGFlow is locally deployed, ensure that your RAGFlow and Ollama are in the same LAN. 
+- If you are using our online demo, ensure that the IP address of your Ollama server is public and accessible.
+
+### 6. How to configure RAGFlow to respond with 100% matched results, rather than utilizing LLM?
+
+1. Click the **Knowledge Base** tab in the middle top of the page.
+2. Right click the desired knowledge base to display the **Configuration** dialogue. 
+3. Choose **Q&A** as the chunk method and click **Save** to confirm your change. 
+
+### Do I need to connect to Redis?
+
+No, connecting to Redis is not required to use RAGFlow. 

rag/app/book.py

@@ -11,6 +11,7 @@
 #  limitations under the License.
 #
 import copy
+from tika import parser
 import re
 from io import BytesIO
 
@@ -37,7 +38,7 @@ class Pdf(PdfParser):
         start = timer()
         self._layouts_rec(zoomin)
         callback(0.67, "Layout analysis finished")
-        print("paddle layouts:", timer() - start)
+        print("layouts:", timer() - start)
         self._table_transformer_job(zoomin)
         callback(0.68, "Table analysis finished")
         self._text_merge()
@@ -67,7 +68,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
     doc["title_sm_tks"] = huqie.qieqie(doc["title_tks"])
     pdf_parser = None
     sections, tbls = [], []
-    if re.search(r"\.docx?$", filename, re.IGNORECASE):
+    if re.search(r"\.docx$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         doc_parser = DocxParser()
         # TODO: table of contents need to be removed
@@ -75,6 +76,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
             binary if binary else filename, from_page=from_page, to_page=to_page)
         remove_contents_table(sections, eng=is_english(
             random_choices([t for t, _ in sections], k=200)))
+        tbls = [((None, lns), None) for lns in tbls]
         callback(0.8, "Finish parsing.")
 
     elif re.search(r"\.pdf$", filename, re.IGNORECASE):
@@ -103,9 +105,19 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
             random_choices([t for t, _ in sections], k=200)))
         callback(0.8, "Finish parsing.")
 
+    elif re.search(r"\.doc$", filename, re.IGNORECASE):
+        callback(0.1, "Start to parse.")
+        binary = BytesIO(binary)
+        doc_parsed = parser.from_buffer(binary)
+        sections = doc_parsed['content'].split('\n')
+        sections = [(l, "") for l in sections if l]
+        remove_contents_table(sections, eng=is_english(
+            random_choices([t for t, _ in sections], k=200)))
+        callback(0.8, "Finish parsing.")
+
     else:
         raise NotImplementedError(
-            "file type not supported yet(docx, pdf, txt supported)")
+            "file type not supported yet(doc, docx, pdf, txt supported)")
 
     make_colon_as_title(sections)
     bull = bullets_category(

rag/app/laws.py

@@ -11,6 +11,7 @@
 #  limitations under the License.
 #
 import copy
+from tika import parser
 import re
 from io import BytesIO
 from docx import Document
@@ -71,7 +72,7 @@ class Pdf(PdfParser):
         start = timer()
         self._layouts_rec(zoomin)
         callback(0.67, "Layout analysis finished")
-        cron_logger.info("paddle layouts:".format(
+        cron_logger.info("layouts:".format(
             (timer() - start) / (self.total_page + 0.1)))
         self._naive_vertical_merge()
 
@@ -93,7 +94,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
     doc["title_sm_tks"] = huqie.qieqie(doc["title_tks"])
     pdf_parser = None
     sections = []
-    if re.search(r"\.docx?$", filename, re.IGNORECASE):
+    if re.search(r"\.docx$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         for txt in Docx()(filename, binary):
             sections.append(txt)
@@ -123,9 +124,18 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         sections = txt.split("\n")
         sections = [l for l in sections if l]
         callback(0.8, "Finish parsing.")
+
+    elif re.search(r"\.doc$", filename, re.IGNORECASE):
+        callback(0.1, "Start to parse.")
+        binary = BytesIO(binary)
+        doc_parsed = parser.from_buffer(binary)
+        sections = doc_parsed['content'].split('\n')
+        sections = [l for l in sections if l]
+        callback(0.8, "Finish parsing.")
+
     else:
         raise NotImplementedError(
-            "file type not supported yet(docx, pdf, txt supported)")
+            "file type not supported yet(doc, docx, pdf, txt supported)")
 
     # is it English
     eng = lang.lower() == "english"  # is_english(sections)

rag/app/manual.py

@@ -32,7 +32,7 @@ class Pdf(PdfParser):
 
         self._layouts_rec(zoomin)
         callback(0.65, "Layout analysis finished.")
-        print("paddle layouts:", timer() - start)
+        print("layouts:", timer() - start)
         self._table_transformer_job(zoomin)
         callback(0.67, "Table analysis finished.")
         self._text_merge()

rag/app/naive.py

@@ -10,8 +10,10 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+from tika import parser
 from io import BytesIO
 from docx import Document
+from timeit import default_timer as timer
 import re
 from deepdoc.parser.pdf_parser import PlainParser
 from rag.nlp import huqie, naive_merge, tokenize_table, tokenize_chunks, find_codec
@@ -66,6 +68,7 @@ class Docx(DocxParser):
 class Pdf(PdfParser):
     def __call__(self, filename, binary=None, from_page=0,
                  to_page=100000, zoomin=3, callback=None):
+        start = timer()
         callback(msg="OCR is  running...")
         self.__images__(
             filename if not binary else binary,
@@ -75,12 +78,11 @@ class Pdf(PdfParser):
             callback
         )
         callback(msg="OCR finished")
+        cron_logger.info("OCR({}~{}): {}".format(from_page, to_page, timer() - start))
 
-        from timeit import default_timer as timer
         start = timer()
         self._layouts_rec(zoomin)
         callback(0.63, "Layout analysis finished.")
-        print("paddle layouts:", timer() - start)
         self._table_transformer_job(zoomin)
         callback(0.65, "Table analysis finished.")
         self._text_merge()
@@ -90,8 +92,7 @@ class Pdf(PdfParser):
         self._concat_downward()
         #self._filter_forpages()
 
-        cron_logger.info("paddle layouts:".format(
+        cron_logger.info("layouts: {}".format(timer() - start))
-            (timer() - start) / (self.total_page + 0.1)))
         return [(b["text"], self._line_tag(b, zoomin))
                 for b in self.boxes], tbls
 
@@ -117,7 +118,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
     res = []
     pdf_parser = None
     sections = []
-    if re.search(r"\.docx?$", filename, re.IGNORECASE):
+    if re.search(r"\.docx$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         sections, tbls = Docx()(filename, binary)
         res = tokenize_table(tbls, doc, eng)
@@ -135,7 +136,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         excel_parser = ExcelParser()
         sections = [(excel_parser.html(binary), "")]
 
-    elif re.search(r"\.txt$", filename, re.IGNORECASE):
+    elif re.search(r"\.(txt|md)$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         txt = ""
         if binary:
@@ -152,16 +153,26 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         sections = [(l, "") for l in sections if l]
         callback(0.8, "Finish parsing.")
 
+    elif re.search(r"\.doc$", filename, re.IGNORECASE):
+        callback(0.1, "Start to parse.")
+        binary = BytesIO(binary)
+        doc_parsed = parser.from_buffer(binary)
+        sections = doc_parsed['content'].split('\n')
+        sections = [(l, "") for l in sections if l]
+        callback(0.8, "Finish parsing.")
+
     else:
         raise NotImplementedError(
-            "file type not supported yet(docx, pdf, txt supported)")
+            "file type not supported yet(doc, docx, pdf, txt supported)")
 
+    st = timer()
     chunks = naive_merge(
         sections, parser_config.get(
             "chunk_token_num", 128), parser_config.get(
             "delimiter", "\n!?。；！？"))
 
     res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser))
+    cron_logger.info("naive_merge({}): {}".format(filename, timer() - st))
     return res
 
 

rag/app/one.py

@@ -10,6 +10,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+from tika import parser
+from io import BytesIO
 import re
 from rag.app import laws
 from rag.nlp import huqie, tokenize, find_codec
@@ -33,7 +35,7 @@ class Pdf(PdfParser):
         start = timer()
         self._layouts_rec(zoomin, drop=False)
         callback(0.63, "Layout analysis finished.")
-        print("paddle layouts:", timer() - start)
+        print("layouts:", timer() - start)
         self._table_transformer_job(zoomin)
         callback(0.65, "Table analysis finished.")
         self._text_merge()
@@ -60,7 +62,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
 
     eng = lang.lower() == "english"  # is_english(cks)
 
-    if re.search(r"\.docx?$", filename, re.IGNORECASE):
+    if re.search(r"\.docx$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         sections = [txt for txt in laws.Docx()(filename, binary) if txt]
         callback(0.8, "Finish parsing.")
@@ -95,9 +97,17 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         sections = [s for s in sections if s]
         callback(0.8, "Finish parsing.")
 
+    elif re.search(r"\.doc$", filename, re.IGNORECASE):
+        callback(0.1, "Start to parse.")
+        binary = BytesIO(binary)
+        doc_parsed = parser.from_buffer(binary)
+        sections = doc_parsed['content'].split('\n')
+        sections = [l for l in sections if l]
+        callback(0.8, "Finish parsing.")
+
     else:
         raise NotImplementedError(
-            "file type not supported yet(docx, pdf, txt supported)")
+            "file type not supported yet(doc, docx, pdf, txt supported)")
 
     doc = {
         "docnm_kwd": filename,

rag/app/paper.py

@@ -42,7 +42,7 @@ class Pdf(PdfParser):
         start = timer()
         self._layouts_rec(zoomin)
         callback(0.63, "Layout analysis finished")
-        print("paddle layouts:", timer() - start)
+        print("layouts:", timer() - start)
         self._table_transformer_job(zoomin)
         callback(0.68, "Table analysis finished")
         self._text_merge()
@@ -78,7 +78,7 @@ class Pdf(PdfParser):
         title = ""
         authors = []
         i = 0
-        while i < min(32, len(self.boxes)):
+        while i < min(32, len(self.boxes)-1):
             b = self.boxes[i]
             i += 1
             if b.get("layoutno", "").find("title") >= 0:

rag/llm/__init__.py

@@ -25,7 +25,7 @@ EmbeddingModel = {
     "Tongyi-Qianwen": HuEmbedding, #QWenEmbed,
     "ZHIPU-AI": ZhipuEmbed,
     "FastEmbed": FastEmbed,
-    "QAnything": QAnythingEmbed
+    "Youdao": YoudaoEmbed
 }
 
 

rag/llm/chat_model.py

@@ -153,7 +153,7 @@ class OllamaChat(Base):
                 options=options
             )
             ans = response["message"]["content"].strip()
-            return ans, response["eval_count"] + response["prompt_eval_count"]
+            return ans, response["eval_count"] + response.get("prompt_eval_count", 0)
         except Exception as e:
             return "**ERROR**: " + str(e), 0
 

rag/llm/embedding_model.py

@@ -229,19 +229,19 @@ class XinferenceEmbed(Base):
         return np.array(res.data[0].embedding), res.usage.total_tokens
 
 
-class QAnythingEmbed(Base):
+class YoudaoEmbed(Base):
     _client = None
 
     def __init__(self, key=None, model_name="maidalun1020/bce-embedding-base_v1", **kwargs):
         from BCEmbedding import EmbeddingModel as qanthing
-        if not QAnythingEmbed._client:
+        if not YoudaoEmbed._client:
             try:
                 print("LOADING BCE...")
-                QAnythingEmbed._client = qanthing(model_name_or_path=os.path.join(
+                YoudaoEmbed._client = qanthing(model_name_or_path=os.path.join(
                     get_project_base_directory(),
                     "rag/res/bce-embedding-base_v1"))
             except Exception as e:
-                QAnythingEmbed._client = qanthing(
+                YoudaoEmbed._client = qanthing(
                     model_name_or_path=model_name.replace(
                         "maidalun1020", "InfiniFlow"))
 
@@ -251,10 +251,10 @@ class QAnythingEmbed(Base):
         for t in texts:
             token_count += num_tokens_from_string(t)
         for i in range(0, len(texts), batch_size):
-            embds = QAnythingEmbed._client.encode(texts[i:i + batch_size])
+            embds = YoudaoEmbed._client.encode(texts[i:i + batch_size])
             res.extend(embds)
         return np.array(res), token_count
 
     def encode_queries(self, text):
-        embds = QAnythingEmbed._client.encode([text])
+        embds = YoudaoEmbed._client.encode([text])
         return np.array(embds[0]), num_tokens_from_string(text)

rag/settings.py

@@ -25,6 +25,11 @@ SUBPROCESS_STD_LOG_NAME = "std.log"
 
 ES = get_base_config("es", {})
 MINIO = decrypt_database_config(name="minio")
+try:
+    REDIS = decrypt_database_config(name="redis")
+except Exception as e:
+    REDIS = {}
+    pass
 DOC_MAXIMUM_SIZE = 128 * 1024 * 1024
 
 # Logger
@@ -39,5 +44,6 @@ LoggerFactory.LEVEL = 30
 es_logger = getLogger("es")
 minio_logger = getLogger("minio")
 cron_logger = getLogger("cron_logger")
+cron_logger.setLevel(20)
 chunk_logger = getLogger("chunk_logger")
 database_logger = getLogger("database")

rag/svr/cache_file_svr.py

@@ -0,0 +1,43 @@
+import random
+import time
+import traceback
+
+from api.db.db_models import close_connection
+from api.db.services.task_service import TaskService
+from rag.utils import MINIO
+from rag.utils.redis_conn import REDIS_CONN
+
+
+def collect():
+    doc_locations = TaskService.get_ongoing_doc_name()
+    #print(tasks)
+    if len(doc_locations) == 0:
+        time.sleep(1)
+        return
+    return doc_locations
+
+def main():
+    locations = collect()
+    if not locations:return
+    print("TASKS:", len(locations))
+    for kb_id, loc in locations:
+        try:
+            if REDIS_CONN.is_alive():
+                try:
+                    key = "{}/{}".format(kb_id, loc)
+                    if REDIS_CONN.exist(key):continue
+                    file_bin = MINIO.get(kb_id, loc)
+                    REDIS_CONN.transaction(key, file_bin, 12 * 60)
+                    print("CACHE:", loc)
+                except Exception as e:
+                    traceback.print_stack(e)
+        except Exception as e:
+            traceback.print_stack(e)
+
+
+
+if __name__ == "__main__":
+    while True:
+        main()
+        close_connection()
+        time.sleep(1)

rag/svr/task_broker.py

@@ -32,6 +32,9 @@ from api.db.services.document_service import DocumentService
 from api.settings import database_logger
 from api.utils import get_format_time, get_uuid
 from api.utils.file_utils import get_project_base_directory
+from rag.utils.redis_conn import REDIS_CONN
+from api.db.db_models import init_database_tables as init_web_db
+from api.db.init_data import init_web_data
 
 
 def collect(tm):
@@ -84,10 +87,16 @@ def dispatch():
 
         tsks = []
         try:
+            file_bin = MINIO.get(r["kb_id"], r["location"])
+            if REDIS_CONN.is_alive():
+                try:
+                    REDIS_CONN.set("{}/{}".format(r["kb_id"], r["location"]), file_bin, 12*60)
+                except Exception as e:
+                    cron_logger.warning("Put into redis[EXCEPTION]:" + str(e))
+
             if r["type"] == FileType.PDF.value:
                 do_layout = r["parser_config"].get("layout_recognize", True)
-                pages = PdfParser.total_page_number(
+                pages = PdfParser.total_page_number(r["name"], file_bin)
-                        r["name"], MINIO.get(r["kb_id"], r["location"]))
                 page_size = r["parser_config"].get("task_page_size", 12)
                 if r["parser_id"] == "paper":
                     page_size = r["parser_config"].get("task_page_size", 22)
@@ -110,8 +119,7 @@ def dispatch():
 
             elif r["parser_id"] == "table":
                 rn = HuExcelParser.row_number(
-                    r["name"], MINIO.get(
+                    r["name"], file_bin)
-                        r["kb_id"], r["location"]))
                 for i in range(0, rn, 3000):
                     task = new_task()
                     task["from_page"] = i
@@ -159,7 +167,7 @@ def update_progress():
             info = {
                 "process_duation": datetime.timestamp(
                     datetime.now()) -
-                d["process_begin_at"].timestamp(),
+                                   d["process_begin_at"].timestamp(),
                 "run": status}
             if prg != 0:
                 info["progress"] = prg
@@ -175,6 +183,9 @@ if __name__ == "__main__":
     peewee_logger.propagate = False
     peewee_logger.addHandler(database_logger.handlers[0])
     peewee_logger.setLevel(database_logger.level)
+    # init db
+    init_web_db()
+    init_web_data()
 
     while True:
         dispatch()

rag/svr/task_executor.py

@@ -19,13 +19,12 @@ import logging
 import os
 import hashlib
 import copy
-import random
 import re
 import sys
 import time
 import traceback
 from functools import partial
-
+from rag.utils import MINIO
 from api.db.db_models import close_connection
 from rag.settings import database_logger
 from rag.settings import cron_logger, DOC_MAXIMUM_SIZE
@@ -35,7 +34,7 @@ from elasticsearch_dsl import Q
 from multiprocessing.context import TimeoutError
 from api.db.services.task_service import TaskService
 from rag.utils import ELASTICSEARCH
-from rag.utils import MINIO
+from timeit import default_timer as timer
 from rag.utils import rmSpace, findMaxTm
 
 from rag.nlp import search
@@ -48,6 +47,7 @@ from api.db import LLMType, ParserType
 from api.db.services.document_service import DocumentService
 from api.db.services.llm_service import LLMBundle
 from api.utils.file_utils import get_project_base_directory
+from rag.utils.redis_conn import REDIS_CONN
 
 BATCH_SIZE = 64
 
@@ -105,11 +105,22 @@ def collect(comm, mod, tm):
 
 def get_minio_binary(bucket, name):
     global MINIO
+    if REDIS_CONN.is_alive():
+        try:
+            for _ in range(30):
+                if REDIS_CONN.exist("{}/{}".format(bucket, name)):
+                    time.sleep(1)
+                    break
+                time.sleep(1)
+            r = REDIS_CONN.get("{}/{}".format(bucket, name))
+            if r: return r
+            cron_logger.warning("Cache missing: {}".format(name))
+        except Exception as e:
+            cron_logger.warning("Get redis[EXCEPTION]:" + str(e))
     return MINIO.get(bucket, name)
 
 
 def build(row):
-    from timeit import default_timer as timer
     if row["size"] > DOC_MAXIMUM_SIZE:
         set_progress(row["id"], prog=-1, msg="File size exceeds( <= %dMb )" %
                      (int(DOC_MAXIMUM_SIZE / 1024 / 1024)))
@@ -158,6 +169,7 @@ def build(row):
         "doc_id": row["doc_id"],
         "kb_id": [str(row["kb_id"])]
     }
+    el = 0
     for ck in cks:
         d = copy.deepcopy(doc)
         d.update(ck)
@@ -177,10 +189,13 @@ def build(row):
         else:
             d["image"].save(output_buffer, format='JPEG')
 
+        st = timer()
         MINIO.put(row["kb_id"], d["_id"], output_buffer.getvalue())
+        el += timer() - st
         d["img_id"] = "{}-{}".format(row["kb_id"], d["_id"])
         del d["image"]
         docs.append(d)
+    cron_logger.info("MINIO PUT({}):{}".format(row["name"], el))
 
     return docs
 
@@ -253,7 +268,9 @@ def main(comm, mod):
             callback(prog=-1, msg=str(e))
             continue
 
+        st = timer()
         cks = build(r)
+        cron_logger.info("Build chunks({}): {}".format(r["name"], timer()-st))
         if cks is None:
             continue
         if not cks:
@@ -265,17 +282,21 @@ def main(comm, mod):
         callback(
             msg="Finished slicing files(%d). Start to embedding the content." %
             len(cks))
+        st = timer()
         try:
             tk_count = embedding(cks, embd_mdl, r["parser_config"], callback)
         except Exception as e:
             callback(-1, "Embedding error:{}".format(str(e)))
             cron_logger.error(str(e))
             tk_count = 0
+        cron_logger.info("Embedding elapsed({}): {}".format(r["name"], timer()-st))
 
-        callback(msg="Finished embedding! Start to build index!")
+        callback(msg="Finished embedding({})! Start to build index!".format(timer()-st))
         init_kb(r)
         chunk_count = len(set([c["_id"] for c in cks]))
+        st = timer()
         es_r = ELASTICSEARCH.bulk(cks, search.index_name(r["tenant_id"]))
+        cron_logger.info("Indexing elapsed({}): {}".format(r["name"], timer()-st))
         if es_r:
             callback(-1, "Index failure!")
             ELASTICSEARCH.deleteByQuery(
@@ -290,8 +311,8 @@ def main(comm, mod):
             DocumentService.increment_chunk_num(
                 r["doc_id"], r["kb_id"], tk_count, chunk_count, 0)
             cron_logger.info(
-                "Chunk doc({}), token({}), chunks({})".format(
+                "Chunk doc({}), token({}), chunks({}), elapsed:{}".format(
-                    r["id"], tk_count, len(cks)))
+                    r["id"], tk_count, len(cks), timer()-st))
 
         tmf.write(str(r["update_time"]) + "\n")
     tmf.close()

rag/utils/minio_conn.py

@@ -56,7 +56,6 @@ class HuMinio(object):
         except Exception as e:
             minio_logger.error(f"Fail rm {bucket}/{fnm}: " + str(e))
 
-
     def get(self, bucket, fnm):
         for _ in range(1):
             try:

rag/utils/redis_conn.py

@@ -0,0 +1,74 @@
+import json
+
+import redis
+import logging
+from rag import settings
+from rag.utils import singleton
+
+@singleton
+class RedisDB:
+    def __init__(self):
+        self.REDIS = None
+        self.config = settings.REDIS
+        self.__open__()
+
+    def __open__(self):
+        try:
+            self.REDIS = redis.Redis(host=self.config.get("host", "redis").split(":")[0],
+                                     port=int(self.config.get("host", ":6379").split(":")[1]),
+                                     db=int(self.config.get("db", 1)),
+                                     password=self.config.get("password"))
+        except Exception as e:
+            logging.warning("Redis can't be connected.")
+        return self.REDIS
+
+    def is_alive(self):
+        return self.REDIS is not None
+
+    def exist(self, k):
+        if not self.REDIS: return
+        try:
+            return self.REDIS.exists(k)
+        except Exception as e:
+            logging.warning("[EXCEPTION]exist" + str(k) + "||" + str(e))
+            self.__open__()
+
+    def get(self, k):
+        if not self.REDIS: return
+        try:
+            return self.REDIS.get(k)
+        except Exception as e:
+            logging.warning("[EXCEPTION]get" + str(k) + "||" + str(e))
+            self.__open__()
+
+    def set_obj(self, k, obj, exp=3600):
+        try:
+            self.REDIS.set(k, json.dumps(obj, ensure_ascii=False), exp)
+            return True
+        except Exception as e:
+            logging.warning("[EXCEPTION]set_obj" + str(k) + "||" + str(e))
+            self.__open__()
+        return False
+
+    def set(self, k, v, exp=3600):
+        try:
+            self.REDIS.set(k, v, exp)
+            return True
+        except Exception as e:
+            logging.warning("[EXCEPTION]set" + str(k) + "||" + str(e))
+            self.__open__()
+        return False
+
+    def transaction(self, key, value, exp=3600):
+        try:
+            pipeline = self.REDIS.pipeline(transaction=True)
+            pipeline.set(key, value, exp, nx=True)
+            pipeline.execute()
+            return True
+        except Exception as e:
+            logging.warning("[EXCEPTION]set" + str(key) + "||" + str(e))
+            self.__open__()
+        return False
+
+
+REDIS_CONN = RedisDB()

requirements.txt

@@ -116,6 +116,7 @@ sniffio==1.3.1
 StrEnum==0.4.15
 sympy==1.12
 threadpoolctl==3.3.0
+tika==2.6.0
 tiktoken==0.6.0
 tokenizers==0.15.2
 torch==2.2.1
@@ -133,4 +134,4 @@ xxhash==3.4.1
 yarl==1.9.4
 zhipuai==2.0.1
 BCEmbedding
-loguru==0.7.2
+loguru==0.7.2

web/.umirc.ts

@@ -27,7 +27,7 @@ export default defineConfig({
   devtool: 'source-map',
   proxy: {
     '/v1': {
-      target: 'http://123.60.95.134:9380/',
+      target: 'http://192.168.200.233:9380/',
       changeOrigin: true,
       // pathRewrite: { '^/v1': '/v1' },
     },

web/externals.d.ts

@@ -0,0 +1,138 @@
+// This file is generated by Umi automatically
+// DO NOT CHANGE IT MANUALLY!
+type CSSModuleClasses = { readonly [key: string]: string };
+declare module '*.css' {
+  const classes: CSSModuleClasses;
+  export default classes;
+}
+declare module '*.scss' {
+  const classes: CSSModuleClasses;
+  export default classes;
+}
+declare module '*.sass' {
+  const classes: CSSModuleClasses;
+  export default classes;
+}
+declare module '*.less' {
+  const classes: CSSModuleClasses;
+  export default classes;
+}
+declare module '*.styl' {
+  const classes: CSSModuleClasses;
+  export default classes;
+}
+declare module '*.stylus' {
+  const classes: CSSModuleClasses;
+  export default classes;
+}
+
+// images
+declare module '*.jpg' {
+  const src: string;
+  export default src;
+}
+declare module '*.jpeg' {
+  const src: string;
+  export default src;
+}
+declare module '*.png' {
+  const src: string;
+  export default src;
+}
+declare module '*.gif' {
+  const src: string;
+  export default src;
+}
+declare module '*.svg' {
+  import * as React from 'react';
+  export const ReactComponent: React.FunctionComponent<
+    React.SVGProps<SVGSVGElement> & { title?: string }
+  >;
+
+  const src: string;
+  export default src;
+}
+declare module '*.ico' {
+  const src: string;
+  export default src;
+}
+declare module '*.webp' {
+  const src: string;
+  export default src;
+}
+declare module '*.avif' {
+  const src: string;
+  export default src;
+}
+
+// media
+declare module '*.mp4' {
+  const src: string;
+  export default src;
+}
+declare module '*.webm' {
+  const src: string;
+  export default src;
+}
+declare module '*.ogg' {
+  const src: string;
+  export default src;
+}
+declare module '*.mp3' {
+  const src: string;
+  export default src;
+}
+declare module '*.wav' {
+  const src: string;
+  export default src;
+}
+declare module '*.flac' {
+  const src: string;
+  export default src;
+}
+declare module '*.aac' {
+  const src: string;
+  export default src;
+}
+
+// fonts
+declare module '*.woff' {
+  const src: string;
+  export default src;
+}
+declare module '*.woff2' {
+  const src: string;
+  export default src;
+}
+declare module '*.eot' {
+  const src: string;
+  export default src;
+}
+declare module '*.ttf' {
+  const src: string;
+  export default src;
+}
+declare module '*.otf' {
+  const src: string;
+  export default src;
+}
+
+// other
+declare module '*.wasm' {
+  const initWasm: (
+    options: WebAssembly.Imports,
+  ) => Promise<WebAssembly.Exports>;
+  export default initWasm;
+}
+declare module '*.webmanifest' {
+  const src: string;
+  export default src;
+}
+declare module '*.pdf' {
+  const src: string;
+  export default src;
+}
+declare module '*.txt' {
+  const src: string;
+  export default src;
+}

web/src/assets/svg/file-icon/folder.svg

@@ -0,0 +1,18 @@
+<svg width="24" height="18" viewBox="0 0 24 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path
+        d="M1.32202e-08 2.54731L21.5 2.54731C22.8807 2.54731 24 3.4977 24 4.67006L24 15.2838C24 16.4562 22.8807 17.4066 21.5 17.4066L12 17.4066L2.5 17.4066C1.11929 17.4066 8.54054e-08 16.4562 7.9321e-08 15.2838L1.32202e-08 2.54731Z"
+        fill="#FBBC1A" />
+    <path
+        d="M2.97454e-08 5.73144L7.49143e-08 14.4347C8.09987e-08 15.6071 1.11929 16.5575 2.5 16.5575L21.5 16.5575C22.8807 16.5575 24 15.6071 24 14.4347L24 5.51916C24 4.3468 22.8807 3.39641 21.5 3.39641L11 3.39641L11 4.45779C11 5.16121 10.3284 5.73144 9.5 5.73144L2.97454e-08 5.73144Z"
+        fill="url(#paint0_linear_2323_8307)" />
+    <path
+        d="M8.81345e-09 1.6982C3.94591e-09 0.760312 0.89543 -4.64716e-09 2 -1.03797e-08L9 -4.67088e-08C10.1046 -5.24413e-08 11 0.760312 11 1.6982L11 2.54731L1.32202e-08 2.54731L8.81345e-09 1.6982Z"
+        fill="#FBBC1A" />
+    <defs>
+        <linearGradient id="paint0_linear_2323_8307" x1="0" y1="0" x2="28.8004" y2="20.3231"
+            gradientUnits="userSpaceOnUse">
+            <stop stop-color="#FFE69C" />
+            <stop offset="1" stop-color="#FFC937" />
+        </linearGradient>
+    </defs>
+</svg>

web/src/base.ts

@@ -0,0 +1,48 @@
+import isObject from 'lodash/isObject';
+import { DvaModel } from 'umi';
+import { BaseState } from './interfaces/common';
+
+type State = Record<string, any>;
+type DvaModelKey<T> = keyof DvaModel<T>;
+
+export const modelExtend = <T>(
+  baseModel: Partial<DvaModel<any>>,
+  extendModel: DvaModel<any>,
+): DvaModel<T> => {
+  return Object.keys(extendModel).reduce<DvaModel<T>>((pre, cur) => {
+    const baseValue = baseModel[cur as DvaModelKey<State>];
+    const value = extendModel[cur as DvaModelKey<State>];
+
+    if (isObject(value) && isObject(baseValue) && typeof value !== 'string') {
+      const key = cur as Exclude<DvaModelKey<State>, 'namespace'>;
+
+      pre[key] = {
+        ...baseValue,
+        ...value,
+      } as any;
+    } else {
+      pre[cur as DvaModelKey<State>] = value as any;
+    }
+
+    return pre;
+  }, {} as DvaModel<T>);
+};
+
+export const paginationModel: Partial<DvaModel<BaseState>> = {
+  state: {
+    searchString: '',
+    pagination: {
+      total: 0,
+      current: 1,
+      pageSize: 10,
+    },
+  },
+  reducers: {
+    setSearchString(state, { payload }) {
+      return { ...state, searchString: payload };
+    },
+    setPagination(state, { payload }) {
+      return { ...state, pagination: { ...state.pagination, ...payload } };
+    },
+  },
+};

web/src/hooks/fileManagerHooks.ts

@@ -0,0 +1,144 @@
+import {
+  IConnectRequestBody,
+  IFileListRequestBody,
+} from '@/interfaces/request/file-manager';
+import { UploadFile } from 'antd';
+import { useCallback } from 'react';
+import { useDispatch, useSelector } from 'umi';
+
+export const useFetchFileList = () => {
+  const dispatch = useDispatch();
+
+  const fetchFileList = useCallback(
+    (payload: IFileListRequestBody) => {
+      return dispatch<any>({
+        type: 'fileManager/listFile',
+        payload,
+      });
+    },
+    [dispatch],
+  );
+
+  return fetchFileList;
+};
+
+export const useRemoveFile = () => {
+  const dispatch = useDispatch();
+
+  const removeFile = useCallback(
+    (fileIds: string[], parentId: string) => {
+      return dispatch<any>({
+        type: 'fileManager/removeFile',
+        payload: { fileIds, parentId },
+      });
+    },
+    [dispatch],
+  );
+
+  return removeFile;
+};
+
+export const useRenameFile = () => {
+  const dispatch = useDispatch();
+
+  const renameFile = useCallback(
+    (fileId: string, name: string, parentId: string) => {
+      return dispatch<any>({
+        type: 'fileManager/renameFile',
+        payload: { fileId, name, parentId },
+      });
+    },
+    [dispatch],
+  );
+
+  return renameFile;
+};
+
+export const useFetchParentFolderList = () => {
+  const dispatch = useDispatch();
+
+  const fetchParentFolderList = useCallback(
+    (fileId: string) => {
+      return dispatch<any>({
+        type: 'fileManager/getAllParentFolder',
+        payload: { fileId },
+      });
+    },
+    [dispatch],
+  );
+
+  return fetchParentFolderList;
+};
+
+export const useCreateFolder = () => {
+  const dispatch = useDispatch();
+
+  const createFolder = useCallback(
+    (parentId: string, name: string) => {
+      return dispatch<any>({
+        type: 'fileManager/createFolder',
+        payload: { parentId, name, type: 'folder' },
+      });
+    },
+    [dispatch],
+  );
+
+  return createFolder;
+};
+
+export const useSelectFileList = () => {
+  const fileList = useSelector((state) => state.fileManager.fileList);
+
+  return fileList;
+};
+
+export const useSelectParentFolderList = () => {
+  const parentFolderList = useSelector(
+    (state) => state.fileManager.parentFolderList,
+  );
+  return parentFolderList.toReversed();
+};
+
+export const useUploadFile = () => {
+  const dispatch = useDispatch();
+
+  const uploadFile = useCallback(
+    (fileList: UploadFile[], parentId: string) => {
+      try {
+        return dispatch<any>({
+          type: 'fileManager/uploadFile',
+          payload: {
+            file: fileList,
+            parentId,
+            path: fileList.map((file) => (file as any).webkitRelativePath),
+          },
+        });
+      } catch (errorInfo) {
+        console.log('Failed:', errorInfo);
+      }
+    },
+    [dispatch],
+  );
+
+  return uploadFile;
+};
+
+export const useConnectToKnowledge = () => {
+  const dispatch = useDispatch();
+
+  const uploadFile = useCallback(
+    (payload: IConnectRequestBody) => {
+      try {
+        return dispatch<any>({
+          type: 'fileManager/connectFileToKnowledge',
+          payload,
+        });
+      } catch (errorInfo) {
+        console.log('Failed:', errorInfo);
+      }
+    },
+    [dispatch],
+  );
+
+  return uploadFile;
+};

web/src/hooks/knowledgeHook.ts

@@ -127,13 +127,13 @@ export const useFetchKnowledgeBaseConfiguration = () => {
 
 export const useFetchKnowledgeList = (
   shouldFilterListWithoutDocument: boolean = false,
-): { list: IKnowledge[]; loading: boolean } => {
+) => {
   const dispatch = useDispatch();
   const loading = useOneNamespaceEffectsLoading('knowledgeModel', ['getList']);
 
   const knowledgeModel = useSelector((state: any) => state.knowledgeModel);
   const { data = [] } = knowledgeModel;
-  const list = useMemo(() => {
+  const list: IKnowledge[] = useMemo(() => {
     return shouldFilterListWithoutDocument
       ? data.filter((x: IKnowledge) => x.chunk_num > 0)
       : data;
@@ -149,7 +149,7 @@ export const useFetchKnowledgeList = (
     fetchList();
   }, [fetchList]);
 
-  return { list, loading };
+  return { list, loading, fetchList };
 };
 
 export const useSelectFileThumbnails = () => {

web/src/hooks/logicHooks.ts

@@ -1,9 +1,12 @@
 import { LanguageTranslationMap } from '@/constants/common';
+import { Pagination } from '@/interfaces/common';
 import { IKnowledgeFile } from '@/interfaces/database/knowledge';
 import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
-import { useCallback, useState } from 'react';
+import { PaginationProps } from 'antd';
+import { useCallback, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
-import { useSetModalState } from './commonHooks';
+import { useDispatch } from 'umi';
+import { useSetModalState, useTranslate } from './commonHooks';
 import { useSetDocumentParser } from './documentHooks';
 import { useOneNamespaceEffectsLoading } from './storeHooks';
 import { useSaveSetting } from './userSettingHook';
@@ -62,3 +65,51 @@ export const useChangeLanguage = () => {
 
   return changeLanguage;
 };
+
+export const useGetPagination = (
+  total: number,
+  page: number,
+  pageSize: number,
+  onPageChange: PaginationProps['onChange'],
+) => {
+  const { t } = useTranslate('common');
+
+  const pagination: PaginationProps = useMemo(() => {
+    return {
+      showQuickJumper: true,
+      total,
+      showSizeChanger: true,
+      current: page,
+      pageSize: pageSize,
+      pageSizeOptions: [1, 2, 10, 20, 50, 100],
+      onChange: onPageChange,
+      showTotal: (total) => `${t('total')} ${total}`,
+    };
+  }, [t, onPageChange, page, pageSize, total]);
+
+  return {
+    pagination,
+  };
+};
+
+export const useSetPagination = (namespace: string) => {
+  const dispatch = useDispatch();
+
+  const setPagination = useCallback(
+    (pageNumber = 1, pageSize?: number) => {
+      const pagination: Pagination = {
+        current: pageNumber,
+      } as Pagination;
+      if (pageSize) {
+        pagination.pageSize = pageSize;
+      }
+      dispatch({
+        type: `${namespace}/setPagination`,
+        payload: pagination,
+      });
+    },
+    [dispatch, namespace],
+  );
+
+  return setPagination;
+};

web/src/interfaces/common.ts

@@ -1,6 +1,7 @@
 export interface Pagination {
   current: number;
   pageSize: number;
+  total: number;
 }
 
 export interface BaseState {

web/src/interfaces/database/file-manager.ts

@@ -0,0 +1,30 @@
+export interface IFile {
+  create_date: string;
+  create_time: number;
+  created_by: string;
+  id: string;
+  kbs_info: { kb_id: string; kb_name: string }[];
+  location: string;
+  name: string;
+  parent_id: string;
+  size: number;
+  tenant_id: string;
+  type: string;
+  update_date: string;
+  update_time: number;
+}
+
+export interface IFolder {
+  create_date: string;
+  create_time: number;
+  created_by: string;
+  id: string;
+  location: string;
+  name: string;
+  parent_id: string;
+  size: number;
+  tenant_id: string;
+  type: string;
+  update_date: string;
+  update_time: number;
+}

web/src/interfaces/request/base.ts

@@ -0,0 +1,7 @@
+export interface IPaginationRequestBody {
+  keywords?: string;
+  page?: number;
+  page_size?: number; // name|create|doc_num|create_time|update_time，default：create_time
+  orderby?: string;
+  desc?: string;
+}

web/src/interfaces/request/file-manager.ts

@@ -0,0 +1,14 @@
+import { IPaginationRequestBody } from './base';
+
+export interface IFileListRequestBody extends IPaginationRequestBody {
+  parent_id?: string; // folder id
+}
+
+interface BaseRequestBody {
+  parentId: string;
+}
+
+export interface IConnectRequestBody extends BaseRequestBody {
+  fileIds: string[];
+  kbIds: string[];
+}

web/src/layouts/components/header/index.tsx

@@ -1,4 +1,5 @@
 import { ReactComponent as StarIon } from '@/assets/svg/chat-star.svg';
+// import { ReactComponent as FileIcon } from '@/assets/svg/file-management.svg';
 import { ReactComponent as KnowledgeBaseIcon } from '@/assets/svg/knowledge-base.svg';
 import { ReactComponent as Logo } from '@/assets/svg/logo.svg';
 import { useTranslate } from '@/hooks/commonHooks';

web/src/locales/en.ts

@@ -70,7 +70,7 @@ export default {
       namePlaceholder: 'Please input name!',
       doc: 'Docs',
       datasetDescription:
-        "Hey, don't forget to adjust the chunk after adding the dataset! 😉",
+        '😉 Questions and answers can only be answered after the parsing is successful.',
       addFile: 'Add file',
       searchFiles: 'Search your files',
       localFiles: 'Local files',
@@ -382,7 +382,7 @@ export default {
       passwordDescription:
         'Please enter your current password to change your password.',
       model: 'Model Providers',
-      modelDescription: 'Manage your account settings and preferences here.',
+      modelDescription: 'Set the model parameter and API Key here.',
       team: 'Team',
       logout: 'Log out',
       username: 'Username',

web/src/locales/zh-traditional.ts

@@ -69,7 +69,7 @@ export default {
       name: '名稱',
       namePlaceholder: '請輸入名稱',
       doc: '文件',
-      datasetDescription: '嘿，添加數據集後別忘了調整解析塊！😉',
+      datasetDescription: '😉 解析成功後才能問答哦。',
       addFile: '新增文件',
       searchFiles: '搜索文件',
       localFiles: '本地文件',
@@ -352,7 +352,7 @@ export default {
       password: '密碼',
       passwordDescription: '請輸入您當前的密碼以更改您的密碼。',
       model: '模型提供商',
-      modelDescription: '在此管理您的帳戶設置和首選項。',
+      modelDescription: '在此設置模型參數和 API Key。',
       team: '團隊',
       logout: '登出',
       username: '使用者名稱',

web/src/locales/zh.ts

@@ -69,7 +69,7 @@ export default {
       name: '名称',
       namePlaceholder: '请输入名称',
       doc: '文档',
-      datasetDescription: '嘿，添加数据集后别忘了调整解析块！ 😉',
+      datasetDescription: '😉 解析成功后才能问答哦。',
       addFile: '新增文件',
       searchFiles: '搜索文件',
       localFiles: '本地文件',
@@ -369,7 +369,7 @@ export default {
       password: '密码',
       passwordDescription: '请输入您当前的密码以更改您的密码。',
       model: '模型提供商',
-      modelDescription: '在此管理您的帐户设置和首选项。',
+      modelDescription: '在此设置模型参数和 API Key。',
       team: '团队',
       logout: '登出',
       username: '用户名',

web/src/pages/add-knowledge/components/knowledge-file/document-toolbar.tsx

@@ -182,7 +182,14 @@ const DocumentToolbar = ({ selectedRowKeys, showCreateModal }: IProps) => {
         ),
       },
     ];
-  }, [handleDelete, handleRunClick, handleCancelClick, t]);
+  }, [
+    handleDelete,
+    handleRunClick,
+    handleCancelClick,
+    t,
+    handleDisableClick,
+    handleEnableClick,
+  ]);
 
   return (
     <div className={styles.filter}>

web/src/pages/file-manager/action-cell/index.tsx

@@ -0,0 +1,101 @@
+import { useTranslate } from '@/hooks/commonHooks';
+import { IFile } from '@/interfaces/database/file-manager';
+import { api_host } from '@/utils/api';
+import { downloadFile } from '@/utils/fileUtil';
+import {
+  DeleteOutlined,
+  DownloadOutlined,
+  EditOutlined,
+  ToolOutlined,
+} from '@ant-design/icons';
+import { Button, Space, Tooltip } from 'antd';
+import { useHandleDeleteFile } from '../hooks';
+
+import styles from './index.less';
+
+interface IProps {
+  record: IFile;
+  setCurrentRecord: (record: any) => void;
+  showRenameModal: (record: IFile) => void;
+  showConnectToKnowledgeModal: (record: IFile) => void;
+  setSelectedRowKeys(keys: string[]): void;
+}
+
+const ActionCell = ({
+  record,
+  setCurrentRecord,
+  showRenameModal,
+  showConnectToKnowledgeModal,
+  setSelectedRowKeys,
+}: IProps) => {
+  const documentId = record.id;
+  const beingUsed = false;
+  const { t } = useTranslate('knowledgeDetails');
+  const { handleRemoveFile } = useHandleDeleteFile(
+    [documentId],
+    setSelectedRowKeys,
+  );
+
+  const onDownloadDocument = () => {
+    downloadFile({
+      url: `${api_host}/document/get/${documentId}`,
+      filename: record.name,
+    });
+  };
+
+  const setRecord = () => {
+    setCurrentRecord(record);
+  };
+
+  const onShowRenameModal = () => {
+    setRecord();
+    showRenameModal(record);
+  };
+
+  const onShowConnectToKnowledgeModal = () => {
+    showConnectToKnowledgeModal(record);
+  };
+
+  return (
+    <Space size={0}>
+      <Button
+        type="text"
+        className={styles.iconButton}
+        onClick={onShowConnectToKnowledgeModal}
+      >
+        <ToolOutlined size={20} />
+      </Button>
+
+      <Tooltip title={t('rename', { keyPrefix: 'common' })}>
+        <Button
+          type="text"
+          disabled={beingUsed}
+          onClick={onShowRenameModal}
+          className={styles.iconButton}
+        >
+          <EditOutlined size={20} />
+        </Button>
+      </Tooltip>
+      <Button
+        type="text"
+        disabled={beingUsed}
+        onClick={handleRemoveFile}
+        className={styles.iconButton}
+      >
+        <DeleteOutlined size={20} />
+      </Button>
+      {record.type !== 'folder' && (
+        <Button
+          type="text"
+          disabled={beingUsed}
+          onClick={onDownloadDocument}
+          className={styles.iconButton}
+        >
+          <DownloadOutlined size={20} />
+        </Button>
+      )}
+    </Space>
+  );
+};
+
+export default ActionCell;

web/src/pages/file-manager/connect-to-knowledge-modal/index.tsx

@@ -0,0 +1,55 @@
+import { useFetchKnowledgeList } from '@/hooks/knowledgeHook';
+import { IModalProps } from '@/interfaces/common';
+import { Form, Modal, Select, SelectProps } from 'antd';
+import { useEffect } from 'react';
+
+const ConnectToKnowledgeModal = ({
+  visible,
+  hideModal,
+  onOk,
+  initialValue,
+}: IModalProps<string[]> & { initialValue: string[] }) => {
+  const [form] = Form.useForm();
+  const { list, fetchList } = useFetchKnowledgeList();
+
+  const options: SelectProps['options'] = list?.map((item) => ({
+    label: item.name,
+    value: item.id,
+  }));
+
+  const handleOk = async () => {
+    const values = await form.getFieldsValue();
+    const knowledgeIds = values.knowledgeIds ?? [];
+    return onOk?.(knowledgeIds);
+  };
+
+  useEffect(() => {
+    if (visible) {
+      form.setFieldValue('knowledgeIds', initialValue);
+      fetchList();
+    }
+  }, [visible, fetchList, initialValue, form]);
+
+  return (
+    <Modal
+      title="Add to Knowledge Base"
+      open={visible}
+      onOk={handleOk}
+      onCancel={hideModal}
+    >
+      <Form form={form}>
+        <Form.Item name="knowledgeIds" noStyle>
+          <Select
+            mode="multiple"
+            allowClear
+            style={{ width: '100%' }}
+            placeholder="Please select"
+            options={options}
+          />
+        </Form.Item>
+      </Form>
+    </Modal>
+  );
+};
+
+export default ConnectToKnowledgeModal;

web/src/pages/file-manager/file-toolbar.tsx

@@ -0,0 +1,156 @@
+import { ReactComponent as DeleteIcon } from '@/assets/svg/delete.svg';
+import { useTranslate } from '@/hooks/commonHooks';
+import {
+  DownOutlined,
+  FileTextOutlined,
+  FolderOpenOutlined,
+  PlusOutlined,
+  SearchOutlined,
+} from '@ant-design/icons';
+import {
+  Breadcrumb,
+  BreadcrumbProps,
+  Button,
+  Dropdown,
+  Flex,
+  Input,
+  MenuProps,
+  Space,
+} from 'antd';
+import { useMemo } from 'react';
+import {
+  useFetchDocumentListOnMount,
+  useHandleDeleteFile,
+  useHandleSearchChange,
+  useSelectBreadcrumbItems,
+} from './hooks';
+
+import { Link } from 'umi';
+import styles from './index.less';
+
+interface IProps {
+  selectedRowKeys: string[];
+  showFolderCreateModal: () => void;
+  showFileUploadModal: () => void;
+  setSelectedRowKeys: (keys: string[]) => void;
+}
+
+const itemRender: BreadcrumbProps['itemRender'] = (
+  currentRoute,
+  params,
+  items,
+) => {
+  const isLast = currentRoute?.path === items[items.length - 1]?.path;
+
+  return isLast ? (
+    <span>{currentRoute.title}</span>
+  ) : (
+    <Link to={`${currentRoute.path}`}>{currentRoute.title}</Link>
+  );
+};
+
+const FileToolbar = ({
+  selectedRowKeys,
+  showFolderCreateModal,
+  showFileUploadModal,
+  setSelectedRowKeys,
+}: IProps) => {
+  const { t } = useTranslate('knowledgeDetails');
+  useFetchDocumentListOnMount();
+  const { handleInputChange, searchString } = useHandleSearchChange();
+  const breadcrumbItems = useSelectBreadcrumbItems();
+
+  const actionItems: MenuProps['items'] = useMemo(() => {
+    return [
+      {
+        key: '1',
+        onClick: showFileUploadModal,
+        label: (
+          <div>
+            <Button type="link">
+              <Space>
+                <FileTextOutlined />
+                {t('localFiles')}
+              </Space>
+            </Button>
+          </div>
+        ),
+      },
+      { type: 'divider' },
+      {
+        key: '2',
+        onClick: showFolderCreateModal,
+        label: (
+          <div>
+            <Button type="link">
+              <FolderOpenOutlined />
+              New Folder
+            </Button>
+          </div>
+        ),
+        // disabled: true,
+      },
+    ];
+  }, [t, showFolderCreateModal, showFileUploadModal]);
+
+  const { handleRemoveFile } = useHandleDeleteFile(
+    selectedRowKeys,
+    setSelectedRowKeys,
+  );
+
+  const disabled = selectedRowKeys.length === 0;
+
+  const items: MenuProps['items'] = useMemo(() => {
+    return [
+      {
+        key: '4',
+        onClick: handleRemoveFile,
+        label: (
+          <Flex gap={10}>
+            <span className={styles.deleteIconWrapper}>
+              <DeleteIcon width={18} />
+            </span>
+            <b>{t('delete', { keyPrefix: 'common' })}</b>
+          </Flex>
+        ),
+      },
+    ];
+  }, [handleRemoveFile, t]);
+
+  return (
+    <div className={styles.filter}>
+      <Breadcrumb items={breadcrumbItems} itemRender={itemRender} />
+      <Space>
+        <Dropdown
+          menu={{ items }}
+          placement="bottom"
+          arrow={false}
+          disabled={disabled}
+        >
+          <Button>
+            <Space>
+              <b> {t('bulk')}</b>
+              <DownOutlined />
+            </Space>
+          </Button>
+        </Dropdown>
+        <Input
+          placeholder={t('searchFiles')}
+          value={searchString}
+          style={{ width: 220 }}
+          allowClear
+          onChange={handleInputChange}
+          prefix={<SearchOutlined />}
+        />
+
+        <Dropdown menu={{ items: actionItems }} trigger={['click']}>
+          <Button type="primary" icon={<PlusOutlined />}>
+            {t('addFile')}
+          </Button>
+        </Dropdown>
+      </Space>
+    </div>
+  );
+};
+
+export default FileToolbar;

web/src/pages/file-manager/file-upload-modal/index.tsx

@@ -0,0 +1,130 @@
+import { IModalProps } from '@/interfaces/common';
+import { InboxOutlined } from '@ant-design/icons';
+import {
+  Flex,
+  Modal,
+  Segmented,
+  Tabs,
+  TabsProps,
+  Upload,
+  UploadFile,
+  UploadProps,
+} from 'antd';
+import { Dispatch, SetStateAction, useState } from 'react';
+
+const { Dragger } = Upload;
+
+const FileUpload = ({
+  directory,
+  fileList,
+  setFileList,
+}: {
+  directory: boolean;
+  fileList: UploadFile[];
+  setFileList: Dispatch<SetStateAction<UploadFile[]>>;
+}) => {
+  const props: UploadProps = {
+    multiple: true,
+    onRemove: (file) => {
+      const index = fileList.indexOf(file);
+      const newFileList = fileList.slice();
+      newFileList.splice(index, 1);
+      setFileList(newFileList);
+    },
+    beforeUpload: (file) => {
+      setFileList((pre) => {
+        return [...pre, file];
+      });
+
+      return false;
+    },
+    directory,
+    fileList,
+  };
+
+  return (
+    <Dragger {...props}>
+      <p className="ant-upload-drag-icon">
+        <InboxOutlined />
+      </p>
+      <p className="ant-upload-text">
+        Click or drag file to this area to upload
+      </p>
+      <p className="ant-upload-hint">
+        Support for a single or bulk upload. Strictly prohibited from uploading
+        company data or other banned files.
+      </p>
+    </Dragger>
+  );
+};
+
+const FileUploadModal = ({
+  visible,
+  hideModal,
+  loading,
+  onOk: onFileUploadOk,
+}: IModalProps<UploadFile[]>) => {
+  const [value, setValue] = useState<string | number>('local');
+  const [fileList, setFileList] = useState<UploadFile[]>([]);
+  const [directoryFileList, setDirectoryFileList] = useState<UploadFile[]>([]);
+
+  const onOk = () => {
+    return onFileUploadOk?.([...fileList, ...directoryFileList]);
+  };
+
+  const items: TabsProps['items'] = [
+    {
+      key: '1',
+      label: 'File',
+      children: (
+        <FileUpload
+          directory={false}
+          fileList={fileList}
+          setFileList={setFileList}
+        ></FileUpload>
+      ),
+    },
+    {
+      key: '2',
+      label: 'Directory',
+      children: (
+        <FileUpload
+          directory
+          fileList={directoryFileList}
+          setFileList={setDirectoryFileList}
+        ></FileUpload>
+      ),
+    },
+  ];
+
+  return (
+    <>
+      <Modal
+        title="File upload"
+        open={visible}
+        onOk={onOk}
+        onCancel={hideModal}
+        confirmLoading={loading}
+      >
+        <Flex gap={'large'} vertical>
+          <Segmented
+            options={[
+              { label: 'Local uploads', value: 'local' },
+              { label: 'S3 uploads', value: 's3' },
+            ]}
+            block
+            value={value}
+            onChange={setValue}
+          />
+          {value === 'local' ? (
+            <Tabs defaultActiveKey="1" items={items} />
+          ) : (
+            'coming soon'
+          )}
+        </Flex>
+      </Modal>
+    </>
+  );
+};
+
+export default FileUploadModal;

web/src/pages/file-manager/folder-create-modal/index.tsx

@@ -0,0 +1,67 @@
+import { IModalManagerChildrenProps } from '@/components/modal-manager';
+import { useTranslate } from '@/hooks/commonHooks';
+import { Form, Input, Modal } from 'antd';
+
+interface IProps extends Omit<IModalManagerChildrenProps, 'showModal'> {
+  loading: boolean;
+  onOk: (name: string) => void;
+}
+
+const FolderCreateModal = ({ visible, hideModal, loading, onOk }: IProps) => {
+  const [form] = Form.useForm();
+  const { t } = useTranslate('common');
+
+  type FieldType = {
+    name?: string;
+  };
+
+  const handleOk = async () => {
+    const ret = await form.validateFields();
+
+    return onOk(ret.name);
+  };
+
+  const handleCancel = () => {
+    hideModal();
+  };
+
+  const onFinish = (values: any) => {
+    console.log('Success:', values);
+  };
+
+  const onFinishFailed = (errorInfo: any) => {
+    console.log('Failed:', errorInfo);
+  };
+
+  return (
+    <Modal
+      title={'New Folder'}
+      open={visible}
+      onOk={handleOk}
+      onCancel={handleCancel}
+      okButtonProps={{ loading }}
+      confirmLoading={loading}
+    >
+      <Form
+        name="basic"
+        labelCol={{ span: 4 }}
+        wrapperCol={{ span: 20 }}
+        style={{ maxWidth: 600 }}
+        onFinish={onFinish}
+        onFinishFailed={onFinishFailed}
+        autoComplete="off"
+        form={form}
+      >
+        <Form.Item<FieldType>
+          label={t('name')}
+          name="name"
+          rules={[{ required: true, message: t('namePlaceholder') }]}
+        >
+          <Input />
+        </Form.Item>
+      </Form>
+    </Modal>
+  );
+};
+
+export default FolderCreateModal;

web/src/pages/file-manager/hooks.ts

@@ -0,0 +1,322 @@
+import { useSetModalState, useShowDeleteConfirm } from '@/hooks/commonHooks';
+import {
+  useConnectToKnowledge,
+  useCreateFolder,
+  useFetchFileList,
+  useFetchParentFolderList,
+  useRemoveFile,
+  useRenameFile,
+  useSelectFileList,
+  useSelectParentFolderList,
+  useUploadFile,
+} from '@/hooks/fileManagerHooks';
+import { useGetPagination, useSetPagination } from '@/hooks/logicHooks';
+import { useOneNamespaceEffectsLoading } from '@/hooks/storeHooks';
+import { IFile } from '@/interfaces/database/file-manager';
+import { PaginationProps } from 'antd';
+import { UploadFile } from 'antd/lib';
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { useDispatch, useNavigate, useSearchParams, useSelector } from 'umi';
+
+export const useGetFolderId = () => {
+  const [searchParams] = useSearchParams();
+  const id = searchParams.get('folderId') as string;
+
+  return id ?? '';
+};
+
+export const useFetchDocumentListOnMount = () => {
+  const fetchDocumentList = useFetchFileList();
+  const fileList = useSelectFileList();
+  const id = useGetFolderId();
+  const { searchString, pagination } = useSelector(
+    (state) => state.fileManager,
+  );
+  const { pageSize, current } = pagination;
+
+  const dispatch = useDispatch();
+
+  useEffect(() => {
+    fetchDocumentList({
+      parent_id: id,
+      keywords: searchString,
+      page_size: pageSize,
+      page: current,
+    });
+  }, [dispatch, fetchDocumentList, id, current, pageSize, searchString]);
+
+  return { fetchDocumentList, fileList };
+};
+
+export const useGetFilesPagination = () => {
+  const { pagination } = useSelector((state) => state.fileManager);
+
+  const setPagination = useSetPagination('fileManager');
+
+  const onPageChange: PaginationProps['onChange'] = useCallback(
+    (pageNumber: number, pageSize: number) => {
+      setPagination(pageNumber, pageSize);
+    },
+    [setPagination],
+  );
+
+  const { pagination: paginationInfo } = useGetPagination(
+    pagination.total,
+    pagination.current,
+    pagination.pageSize,
+    onPageChange,
+  );
+
+  return {
+    pagination: paginationInfo,
+    setPagination,
+  };
+};
+
+export const useHandleSearchChange = () => {
+  const dispatch = useDispatch();
+  const { searchString } = useSelector((state) => state.fileManager);
+  const setPagination = useSetPagination('fileManager');
+
+  const handleInputChange = useCallback(
+    (e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => {
+      const value = e.target.value;
+      dispatch({ type: 'fileManager/setSearchString', payload: value });
+      setPagination();
+    },
+    [setPagination, dispatch],
+  );
+
+  return { handleInputChange, searchString };
+};
+
+export const useGetRowSelection = () => {
+  const [selectedRowKeys, setSelectedRowKeys] = useState<React.Key[]>([]);
+
+  const rowSelection = {
+    selectedRowKeys,
+    onChange: (newSelectedRowKeys: React.Key[]) => {
+      setSelectedRowKeys(newSelectedRowKeys);
+    },
+  };
+
+  return { rowSelection, setSelectedRowKeys };
+};
+
+export const useNavigateToOtherFolder = () => {
+  const navigate = useNavigate();
+  const navigateToOtherFolder = useCallback(
+    (folderId: string) => {
+      navigate(`/file?folderId=${folderId}`);
+    },
+    [navigate],
+  );
+
+  return navigateToOtherFolder;
+};
+
+export const useRenameCurrentFile = () => {
+  const [file, setFile] = useState<IFile>({} as IFile);
+  const {
+    visible: fileRenameVisible,
+    hideModal: hideFileRenameModal,
+    showModal: showFileRenameModal,
+  } = useSetModalState();
+  const renameFile = useRenameFile();
+
+  const onFileRenameOk = useCallback(
+    async (name: string) => {
+      const ret = await renameFile(file.id, name, file.parent_id);
+
+      if (ret === 0) {
+        hideFileRenameModal();
+      }
+    },
+    [renameFile, file, hideFileRenameModal],
+  );
+
+  const loading = useOneNamespaceEffectsLoading('fileManager', ['renameFile']);
+
+  const handleShowFileRenameModal = useCallback(
+    async (record: IFile) => {
+      setFile(record);
+      showFileRenameModal();
+    },
+    [showFileRenameModal],
+  );
+
+  return {
+    fileRenameLoading: loading,
+    initialFileName: file.name,
+    onFileRenameOk,
+    fileRenameVisible,
+    hideFileRenameModal,
+    showFileRenameModal: handleShowFileRenameModal,
+  };
+};
+
+export const useSelectBreadcrumbItems = () => {
+  const parentFolderList = useSelectParentFolderList();
+  const id = useGetFolderId();
+  const fetchParentFolderList = useFetchParentFolderList();
+
+  useEffect(() => {
+    if (id) {
+      fetchParentFolderList(id);
+    }
+  }, [id, fetchParentFolderList]);
+
+  return parentFolderList.length === 1
+    ? []
+    : parentFolderList.map((x) => ({
+        title: x.name === '/' ? 'root' : x.name,
+        path: `/file?folderId=${x.id}`,
+      }));
+};
+
+export const useHandleCreateFolder = () => {
+  const {
+    visible: folderCreateModalVisible,
+    hideModal: hideFolderCreateModal,
+    showModal: showFolderCreateModal,
+  } = useSetModalState();
+  const createFolder = useCreateFolder();
+  const id = useGetFolderId();
+
+  const onFolderCreateOk = useCallback(
+    async (name: string) => {
+      const ret = await createFolder(id, name);
+
+      if (ret === 0) {
+        hideFolderCreateModal();
+      }
+    },
+    [createFolder, hideFolderCreateModal, id],
+  );
+
+  const loading = useOneNamespaceEffectsLoading('fileManager', [
+    'createFolder',
+  ]);
+
+  return {
+    folderCreateLoading: loading,
+    onFolderCreateOk,
+    folderCreateModalVisible,
+    hideFolderCreateModal,
+    showFolderCreateModal,
+  };
+};
+
+export const useHandleDeleteFile = (
+  fileIds: string[],
+  setSelectedRowKeys: (keys: string[]) => void,
+) => {
+  const removeDocument = useRemoveFile();
+  const showDeleteConfirm = useShowDeleteConfirm();
+  const parentId = useGetFolderId();
+
+  const handleRemoveFile = () => {
+    showDeleteConfirm({
+      onOk: async () => {
+        const retcode = await removeDocument(fileIds, parentId);
+        if (retcode === 0) {
+          setSelectedRowKeys([]);
+        }
+        return;
+      },
+    });
+  };
+
+  return { handleRemoveFile };
+};
+
+export const useSelectFileListLoading = () => {
+  return useOneNamespaceEffectsLoading('fileManager', ['listFile']);
+};
+
+export const useHandleUploadFile = () => {
+  const {
+    visible: fileUploadVisible,
+    hideModal: hideFileUploadModal,
+    showModal: showFileUploadModal,
+  } = useSetModalState();
+  const uploadFile = useUploadFile();
+  const id = useGetFolderId();
+
+  const onFileUploadOk = useCallback(
+    async (fileList: UploadFile[]) => {
+      console.info('fileList', fileList);
+      if (fileList.length > 0) {
+        const ret = await uploadFile(fileList, id);
+        console.info(ret);
+        if (ret === 0) {
+          hideFileUploadModal();
+        }
+      }
+    },
+    [uploadFile, hideFileUploadModal, id],
+  );
+
+  const loading = useOneNamespaceEffectsLoading('fileManager', ['uploadFile']);
+
+  return {
+    fileUploadLoading: loading,
+    onFileUploadOk,
+    fileUploadVisible,
+    hideFileUploadModal,
+    showFileUploadModal,
+  };
+};
+
+export const useHandleConnectToKnowledge = () => {
+  const {
+    visible: connectToKnowledgeVisible,
+    hideModal: hideConnectToKnowledgeModal,
+    showModal: showConnectToKnowledgeModal,
+  } = useSetModalState();
+  const connectToKnowledge = useConnectToKnowledge();
+  const id = useGetFolderId();
+  const [record, setRecord] = useState<IFile>({} as IFile);
+
+  const initialValue = useMemo(() => {
+    return Array.isArray(record?.kbs_info)
+      ? record?.kbs_info?.map((x) => x.kb_id)
+      : [];
+  }, [record?.kbs_info]);
+
+  const onConnectToKnowledgeOk = useCallback(
+    async (knowledgeIds: string[]) => {
+      const ret = await connectToKnowledge({
+        parentId: id,
+        fileIds: [record.id],
+        kbIds: knowledgeIds,
+      });
+
+      if (ret === 0) {
+        hideConnectToKnowledgeModal();
+      }
+    },
+    [connectToKnowledge, hideConnectToKnowledgeModal, id, record.id],
+  );
+
+  const loading = useOneNamespaceEffectsLoading('fileManager', [
+    'connectFileToKnowledge',
+  ]);
+
+  const handleShowConnectToKnowledgeModal = useCallback(
+    (record: IFile) => {
+      setRecord(record);
+      showConnectToKnowledgeModal();
+    },
+    [showConnectToKnowledgeModal],
+  );
+
+  return {
+    initialValue,
+    connectToKnowledgeLoading: loading,
+    onConnectToKnowledgeOk,
+    connectToKnowledgeVisible,
+    hideConnectToKnowledgeModal,
+    showConnectToKnowledgeModal: handleShowConnectToKnowledgeModal,
+  };
+};

web/src/pages/file-manager/index.less

@@ -0,0 +1,22 @@
+.fileManagerWrapper {
+  flex-basis: 100%;
+  padding: 32px;
+}
+
+.filter {
+  height: 32px;
+  display: flex;
+  margin: 10px 0;
+  justify-content: space-between;
+  padding: 24px 0;
+  align-items: center;
+}
+
+.deleteIconWrapper {
+  width: 22px;
+  text-align: center;
+}
+
+.linkButton {
+  padding: 0;
+}

web/src/pages/file-manager/index.tsx

@@ -0,0 +1,176 @@
+import { useSelectFileList } from '@/hooks/fileManagerHooks';
+import { IFile } from '@/interfaces/database/file-manager';
+import { formatDate } from '@/utils/date';
+import { Button, Flex, Table } from 'antd';
+import { ColumnsType } from 'antd/es/table';
+import ActionCell from './action-cell';
+import FileToolbar from './file-toolbar';
+import {
+  useGetFilesPagination,
+  useGetRowSelection,
+  useHandleConnectToKnowledge,
+  useHandleCreateFolder,
+  useHandleUploadFile,
+  useNavigateToOtherFolder,
+  useRenameCurrentFile,
+  useSelectFileListLoading,
+} from './hooks';
+
+import RenameModal from '@/components/rename-modal';
+import SvgIcon from '@/components/svg-icon';
+import { getExtension } from '@/utils/documentUtils';
+import ConnectToKnowledgeModal from './connect-to-knowledge-modal';
+import FileUploadModal from './file-upload-modal';
+import FolderCreateModal from './folder-create-modal';
+import styles from './index.less';
+
+const FileManager = () => {
+  const fileList = useSelectFileList();
+  const { rowSelection, setSelectedRowKeys } = useGetRowSelection();
+  const loading = useSelectFileListLoading();
+  const navigateToOtherFolder = useNavigateToOtherFolder();
+  const {
+    fileRenameVisible,
+    fileRenameLoading,
+    hideFileRenameModal,
+    showFileRenameModal,
+    initialFileName,
+    onFileRenameOk,
+  } = useRenameCurrentFile();
+  const {
+    folderCreateModalVisible,
+    showFolderCreateModal,
+    hideFolderCreateModal,
+    folderCreateLoading,
+    onFolderCreateOk,
+  } = useHandleCreateFolder();
+  const {
+    fileUploadVisible,
+    hideFileUploadModal,
+    showFileUploadModal,
+    fileUploadLoading,
+    onFileUploadOk,
+  } = useHandleUploadFile();
+  const {
+    connectToKnowledgeVisible,
+    hideConnectToKnowledgeModal,
+    showConnectToKnowledgeModal,
+    onConnectToKnowledgeOk,
+    initialValue,
+  } = useHandleConnectToKnowledge();
+  const { pagination } = useGetFilesPagination();
+
+  const columns: ColumnsType<IFile> = [
+    {
+      title: 'Name',
+      dataIndex: 'name',
+      key: 'name',
+      render(value, record) {
+        return (
+          <Flex gap={10} align="center">
+            <SvgIcon
+              name={`file-icon/${record.type === 'folder' ? 'folder' : getExtension(value)}`}
+              width={24}
+            ></SvgIcon>
+            {record.type === 'folder' ? (
+              <Button
+                type={'link'}
+                className={styles.linkButton}
+                onClick={() => navigateToOtherFolder(record.id)}
+              >
+                {value}
+              </Button>
+            ) : (
+              value
+            )}
+          </Flex>
+        );
+      },
+    },
+    {
+      title: 'Upload Date',
+      dataIndex: 'create_date',
+      key: 'create_date',
+      render(text) {
+        return formatDate(text);
+      },
+    },
+    {
+      title: 'Knowledge Base',
+      dataIndex: 'kbs_info',
+      key: 'kbs_info',
+      render(value) {
+        return Array.isArray(value)
+          ? value?.map((x) => x.kb_name).join(',')
+          : '';
+      },
+    },
+    {
+      title: 'Location',
+      dataIndex: 'location',
+      key: 'location',
+    },
+    {
+      title: 'Action',
+      dataIndex: 'action',
+      key: 'action',
+      render: (text, record) => (
+        <ActionCell
+          record={record}
+          setCurrentRecord={(record: any) => {
+            console.info(record);
+          }}
+          showRenameModal={showFileRenameModal}
+          showConnectToKnowledgeModal={showConnectToKnowledgeModal}
+          setSelectedRowKeys={setSelectedRowKeys}
+        ></ActionCell>
+      ),
+    },
+  ];
+
+  return (
+    <section className={styles.fileManagerWrapper}>
+      <FileToolbar
+        selectedRowKeys={rowSelection.selectedRowKeys as string[]}
+        showFolderCreateModal={showFolderCreateModal}
+        showFileUploadModal={showFileUploadModal}
+        setSelectedRowKeys={setSelectedRowKeys}
+      ></FileToolbar>
+      <Table
+        dataSource={fileList}
+        columns={columns}
+        rowKey={'id'}
+        rowSelection={rowSelection}
+        loading={loading}
+        pagination={pagination}
+      />
+      <RenameModal
+        visible={fileRenameVisible}
+        hideModal={hideFileRenameModal}
+        onOk={onFileRenameOk}
+        initialName={initialFileName}
+        loading={fileRenameLoading}
+      ></RenameModal>
+      <FolderCreateModal
+        loading={folderCreateLoading}
+        visible={folderCreateModalVisible}
+        hideModal={hideFolderCreateModal}
+        onOk={onFolderCreateOk}
+      ></FolderCreateModal>
+      <FileUploadModal
+        visible={fileUploadVisible}
+        hideModal={hideFileUploadModal}
+        loading={fileUploadLoading}
+        onOk={onFileUploadOk}
+      ></FileUploadModal>
+      <ConnectToKnowledgeModal
+        initialValue={initialValue}
+        visible={connectToKnowledgeVisible}
+        hideModal={hideConnectToKnowledgeModal}
+        onOk={onConnectToKnowledgeOk}
+      ></ConnectToKnowledgeModal>
+    </section>
+  );
+};
+
+export default FileManager;

web/src/pages/file-manager/model.ts

@@ -0,0 +1,145 @@
+import { paginationModel } from '@/base';
+import { BaseState } from '@/interfaces/common';
+import { IFile, IFolder } from '@/interfaces/database/file-manager';
+import fileManagerService from '@/services/fileManagerService';
+import omit from 'lodash/omit';
+import { DvaModel } from 'umi';
+
+export interface FileManagerModelState extends BaseState {
+  fileList: IFile[];
+  parentFolderList: IFolder[];
+}
+
+const model: DvaModel<FileManagerModelState> = {
+  namespace: 'fileManager',
+  state: {
+    fileList: [],
+    parentFolderList: [],
+    ...(paginationModel.state as BaseState),
+  },
+  reducers: {
+    setFileList(state, { payload }) {
+      return { ...state, fileList: payload };
+    },
+    setParentFolderList(state, { payload }) {
+      return { ...state, parentFolderList: payload };
+    },
+    ...paginationModel.reducers,
+  },
+  effects: {
+    *removeFile({ payload = {} }, { call, put }) {
+      const { data } = yield call(fileManagerService.removeFile, {
+        fileIds: payload.fileIds,
+      });
+      const { retcode } = data;
+      if (retcode === 0) {
+        yield put({
+          type: 'listFile',
+          payload: { parentId: payload.parentId },
+        });
+      }
+      return retcode;
+    },
+    *listFile({ payload = {} }, { call, put, select }) {
+      const { searchString, pagination } = yield select(
+        (state: any) => state.fileManager,
+      );
+      const { current, pageSize } = pagination;
+      const { data } = yield call(fileManagerService.listFile, {
+        ...payload,
+        keywords: searchString.trim(),
+        page: current,
+        pageSize,
+      });
+      const { retcode, data: res } = data;
+      if (retcode === 0 && Array.isArray(res.files)) {
+        yield put({
+          type: 'setFileList',
+          payload: res.files,
+        });
+        yield put({
+          type: 'setPagination',
+          payload: { total: res.total },
+        });
+      }
+    },
+    *renameFile({ payload = {} }, { call, put }) {
+      const { data } = yield call(
+        fileManagerService.renameFile,
+        omit(payload, ['parentId']),
+      );
+      if (data.retcode === 0) {
+        yield put({
+          type: 'listFile',
+          payload: { parentId: payload.parentId },
+        });
+      }
+      return data.retcode;
+    },
+    *uploadFile({ payload = {} }, { call, put }) {
+      const fileList = payload.file;
+      const pathList = payload.path;
+      const formData = new FormData();
+      formData.append('parent_id', payload.parentId);
+      // formData.append('file', payload.file);
+      // formData.append('path', payload.path);
+      fileList.forEach((file: any, index: number) => {
+        formData.append('file', file);
+        formData.append('path', pathList[index]);
+      });
+      const { data } = yield call(fileManagerService.uploadFile, formData);
+      if (data.retcode === 0) {
+        yield put({
+          type: 'listFile',
+          payload: { parentId: payload.parentId },
+        });
+      }
+      return data.retcode;
+    },
+    *createFolder({ payload = {} }, { call, put }) {
+      const { data } = yield call(fileManagerService.createFolder, payload);
+      if (data.retcode === 0) {
+        yield put({
+          type: 'listFile',
+          payload: { parentId: payload.parentId },
+        });
+      }
+      return data.retcode;
+    },
+    *getAllParentFolder({ payload = {} }, { call, put }) {
+      const { data } = yield call(
+        fileManagerService.getAllParentFolder,
+        payload,
+      );
+      if (data.retcode === 0) {
+        yield put({
+          type: 'setParentFolderList',
+          payload: data.data?.parent_folders ?? [],
+        });
+      }
+      return data.retcode;
+    },
+    *connectFileToKnowledge({ payload = {} }, { call, put }) {
+      const { data } = yield call(
+        fileManagerService.connectFileToKnowledge,
+        omit(payload, 'parentId'),
+      );
+      if (data.retcode === 0) {
+        yield put({
+          type: 'listFile',
+          payload: { parentId: payload.parentId },
+        });
+      }
+      return data.retcode;
+    },
+  },
+};
+
+// const finalModel = modelExtend<DvaModel<FileManagerModelState & BaseState>>(
+//   paginationModel,
+//   model,
+// );
+
+// console.info(finalModel);
+
+export default model;

web/src/pages/file/index.tsx

@@ -1,50 +0,0 @@
-import { UploadOutlined } from '@ant-design/icons';
-import { Button, Upload } from 'antd';
-import React, { useEffect, useState } from 'react';
-
-const File: React.FC = () => {
-  const [fileList, setFileList] = useState([
-    {
-      uid: '0',
-      name: 'xxx.png',
-      status: 'uploading',
-      percent: 10,
-    },
-  ]);
-  const obj = {
-    uid: '-1',
-    name: 'yyy.png',
-    status: 'done',
-    url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
-    thumbUrl:
-      'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
-  };
-  useEffect(() => {
-    const timer = setInterval(() => {
-      setFileList((fileList: any) => {
-        const percent = fileList[0]?.percent;
-        if (percent + 10 >= 100) {
-          clearInterval(timer);
-          return [obj];
-        }
-        const list = [{ ...fileList[0], percent: percent + 10 }];
-        console.log(list);
-        return list;
-      });
-    }, 300);
-  }, []);
-  return (
-    <>
-      <Upload
-        action="https://run.mocky.io/v3/435e224c-44fb-4773-9faf-380c5e6a2188"
-        listType="picture"
-        fileList={[...fileList]}
-        multiple
-      >
-        <Button icon={<UploadOutlined />}>Upload</Button>
-      </Upload>
-    </>
-  );
-};
-
-export default File;

web/src/pages/knowledge/model.ts

@@ -3,7 +3,7 @@ import kbService from '@/services/kbService';
 import { DvaModel } from 'umi';
 
 export interface KnowledgeModelState {
-  data: any[];
+  data: IKnowledge[];
   knowledge: IKnowledge;
 }
 

web/src/pages/login/index.tsx

@@ -167,20 +167,22 @@ const Login = () => {
                     Sign in with Google
                   </div>
                 </Button> */}
-                <Button
+                {location.host === 'demo.ragflow.io' && (
-                  block
+                  <Button
-                  size="large"
+                    block
-                  onClick={toGoogle}
+                    size="large"
-                  style={{ marginTop: 15 }}
+                    onClick={toGoogle}
-                >
+                    style={{ marginTop: 15 }}
-                  <div>
+                  >
-                    <Icon
+                    <div>
-                      icon="local:github"
+                      <Icon
-                      style={{ verticalAlign: 'middle', marginRight: 5 }}
+                        icon="local:github"
-                    />
+                        style={{ verticalAlign: 'middle', marginRight: 5 }}
-                    Sign in with Github
+                      />
-                  </div>
+                      Sign in with Github
-                </Button>
+                    </div>
+                  </Button>
+                )}
               </>
             )}
           </Form>

web/src/pages/user-setting/setting-model/index.tsx

@@ -228,7 +228,7 @@ const UserSettingModel = () => {
         <section className={styles.modelWrapper}>
           <SettingTitle
             title={t('model')}
-            description={t('profileDescription')}
+            description={t('modelDescription')}
             showRightButton
             clickButton={showSystemSettingModal}
           ></SettingTitle>

web/src/routes.ts

@@ -82,7 +82,7 @@ const routes = [
       },
       {
         path: '/file',
-        component: '@/pages/file',
+        component: '@/pages/file-manager',
       },
     ],
   },

web/src/services/fileManagerService.ts

@@ -0,0 +1,51 @@
+import api from '@/utils/api';
+import registerServer from '@/utils/registerServer';
+import request from '@/utils/request';
+
+const {
+  listFile,
+  removeFile,
+  uploadFile,
+  renameFile,
+  getAllParentFolder,
+  createFolder,
+  connectFileToKnowledge,
+} = api;
+
+const methods = {
+  listFile: {
+    url: listFile,
+    method: 'get',
+  },
+  removeFile: {
+    url: removeFile,
+    method: 'post',
+  },
+  uploadFile: {
+    url: uploadFile,
+    method: 'post',
+  },
+  renameFile: {
+    url: renameFile,
+    method: 'post',
+  },
+  getAllParentFolder: {
+    url: getAllParentFolder,
+    method: 'get',
+  },
+  createFolder: {
+    url: createFolder,
+    method: 'post',
+  },
+  connectFileToKnowledge: {
+    url: connectFileToKnowledge,
+    method: 'post',
+  },
+} as const;
+
+const fileManagerService = registerServer<keyof typeof methods>(
+  methods,
+  request,
+);
+
+export default fileManagerService;

web/src/utils/api.ts

@@ -66,4 +66,13 @@ export default {
   createExternalConversation: `${api_host}/api/new_conversation`,
   getExternalConversation: `${api_host}/api/conversation`,
   completeExternalConversation: `${api_host}/api/completion`,
+
+  // file manager
+  listFile: `${api_host}/file/list`,
+  uploadFile: `${api_host}/file/upload`,
+  removeFile: `${api_host}/file/rm`,
+  renameFile: `${api_host}/file/rename`,
+  getAllParentFolder: `${api_host}/file/all_parent_folder`,
+  createFolder: `${api_host}/file/create`,
+  connectFileToKnowledge: `${api_host}/file2document/convert`,
 };

web/src/utils/commonUtil.ts

@@ -5,11 +5,18 @@ export const isFormData = (data: unknown): data is FormData => {
   return data instanceof FormData;
 };
 
+const excludedFields = ['img2txt_id'];
+
+const isExcludedField = (key: string) => {
+  return excludedFields.includes(key);
+};
+
 export const convertTheKeysOfTheObjectToSnake = (data: unknown) => {
   if (isObject(data) && !isFormData(data)) {
     return Object.keys(data).reduce<Record<string, any>>((pre, cur) => {
       const value = (data as Record<string, any>)[cur];
-      pre[isFormData(value) ? cur : snakeCase(cur)] = value;
+      pre[isFormData(value) || isExcludedField(cur) ? cur : snakeCase(cur)] =
+        value;
       return pre;
     }, {});
   }

web/typings.d.ts

@@ -1,8 +1,39 @@
-import 'umi/typings';
+import { ChunkModelState } from '@/pages/add-knowledge/components/knowledge-chunk/model';
+import { KFModelState } from '@/pages/add-knowledge/components/knowledge-file/model';
+import { KSModelState } from '@/pages/add-knowledge/components/knowledge-setting/model';
+import { TestingModelState } from '@/pages/add-knowledge/components/knowledge-testing/model';
+import { kAModelState } from '@/pages/add-knowledge/model';
+import { ChatModelState } from '@/pages/chat/model';
+import { FileManagerModelState } from '@/pages/file-manager/model';
+import { KnowledgeModelState } from '@/pages/knowledge/model';
+import { LoginModelState } from '@/pages/login/model';
+import { SettingModelState } from '@/pages/user-setting/model';
+
 declare module 'lodash';
 
-// declare type Nullable<T> = T | null; invalid
+function useSelector<TState = RootState, TSelected = unknown>(
+  selector: (state: TState) => TSelected,
+  equalityFn?: (left: TSelected, right: TSelected) => boolean,
+): TSelected;
+
+export interface RootState {
+  // loading: Loading;
+  fileManager: FileManagerModelState;
+  chatModel: ChatModelState;
+  loginModel: LoginModelState;
+  knowledgeModel: KnowledgeModelState;
+  settingModel: SettingModelState;
+  kFModel: KFModelState;
+  kAModel: kAModelState;
+  chunkModel: ChunkModelState;
+  kSModel: KSModelState;
+  testingModel: TestingModelState;
+}
 
 declare global {
   type Nullable<T> = T | null;
 }
+
+declare module 'umi' {
+  export { useSelector };
+}