Fix: Unified terminology to "Pipeline" and optimized related component logic. #9869 (#10394)

### What problem does this PR solve?

Fix: Unified terminology to "Pipeline" and optimized related component
logic. #9869

- Added logic to clear pipeline_id when parseType changes in the chunk
method dialog.
- Fixed an issue in the Tooltip form component that prevented clicks
from triggering saves.
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}  # Use the secret as an environment variable
+          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
           fetch-depth: 0
           fetch-tags: true
 
@@ -69,7 +69,7 @@ jobs:
         # https://github.com/actions/upload-release-asset has been replaced by https://github.com/softprops/action-gh-release
         uses: softprops/action-gh-release@v2
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}  # Use the secret as an environment variable
+          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
           prerelease: ${{ env.PRERELEASE }}
           tag_name: ${{ env.RELEASE_TAG }}
           # The body field does not support environment variable substitution directly.

.github/workflows/tests.yml

@@ -34,10 +34,12 @@ jobs:
       # https://github.com/hmarr/debug-action
       #- uses: hmarr/debug-action@v2
 
-      - name: Ensure workspace ownership
+      - name: Show who triggered this workflow
         run: |
           echo "Workflow triggered by ${{ github.event_name }}"
-          echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
+
+      - name: Ensure workspace ownership
+        run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
 
       # https://github.com/actions/checkout/issues/1781
       - name: Check out code
@@ -46,44 +48,6 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
-      - name: Check workflow duplication
-        if: ${{ !cancelled() && !failure() && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci')) }}
-        run: |
-          if [[ ${{ github.event_name }} != 'pull_request' ]]; then
-            HEAD=$(git rev-parse HEAD)
-            # Find a PR that introduced a given commit
-            gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}"
-            PR_NUMBER=$(gh pr list --search ${HEAD} --state merged --json number --jq .[0].number)
-            echo "HEAD=${HEAD}"
-            echo "PR_NUMBER=${PR_NUMBER}"
-            if [[ -n ${PR_NUMBER} ]]; then
-              PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
-              if [[ -f ${PR_SHA_FP} ]]; then
-                read -r PR_SHA PR_RUN_ID < "${PR_SHA_FP}"
-                # Calculate the hash of the current workspace content
-                HEAD_SHA=$(git rev-parse HEAD^{tree})
-                if [[ ${HEAD_SHA} == ${PR_SHA} ]]; then
-                  echo "Cancel myself since the workspace content hash is the same with PR #${PR_NUMBER} merged. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${PR_RUN_ID} for details."
-                  gh run cancel ${GITHUB_RUN_ID}
-                  while true; do
-                    status=$(gh run view ${GITHUB_RUN_ID} --json status -q .status)
-                    [ "$status" = "completed" ] && break
-                    sleep 5
-                  done
-                  exit 1
-                fi
-              fi
-            fi
-          else
-            PR_NUMBER=${{ github.event.pull_request.number }}
-            PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
-            # Calculate the hash of the current workspace content
-            PR_SHA=$(git rev-parse HEAD^{tree})
-            echo "PR #${PR_NUMBER} workspace content hash: ${PR_SHA}"
-            mkdir -p ${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}
-            echo "${PR_SHA} ${GITHUB_RUN_ID}" > ${PR_SHA_FP}
-          fi
-
       # https://github.com/astral-sh/ruff-action
       - name: Static check with Ruff
         uses: astral-sh/ruff-action@v3
@@ -95,11 +59,11 @@ jobs:
         run: |
           RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
           sudo docker pull ubuntu:22.04
-          sudo DOCKER_BUILDKIT=1 docker build --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+          sudo docker build --progress=plain --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 
       - name: Build ragflow:nightly
         run: |
-          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
+          sudo docker build --progress=plain --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
 
       - name: Start ragflow:nightly-slim
         run: |

README.md

@@ -341,13 +341,11 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
 5. If your operating system does not have jemalloc, please install it as follows:
 
    ```bash
-   # Ubuntu
+   # ubuntu
    sudo apt-get install libjemalloc-dev
-   # CentOS
+   # centos
    sudo yum install jemalloc
-   # OpenSUSE
-   sudo zypper install jemalloc
-   # macOS
+   # mac
    sudo brew install jemalloc
    ```
 

admin/admin_client.py

@@ -1,19 +1,3 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
 import argparse
 import base64
 
@@ -406,22 +390,6 @@ class AdminCLI:
         service_id: int = command['number']
         print(f"Showing service: {service_id}")
 
-        url = f'http://{self.host}:{self.port}/api/v1/admin/services/{service_id}'
-        response = requests.get(url, auth=HTTPBasicAuth(self.admin_account, self.admin_password))
-        res_json = response.json()
-        if response.status_code == 200:
-            res_data = res_json['data']
-            if res_data['alive']:
-                print(f"Service {res_data['service_name']} is alive. Detail:")
-                if isinstance(res_data['message'], str):
-                    print(res_data['message'])
-                else:
-                    self._print_table_simple(res_data['message'])
-            else:
-                print(f"Service {res_data['service_name']} is down. Detail: {res_data['message']}")
-        else:
-            print(f"Fail to show service, code: {res_json['code']}, message: {res_json['message']}")
-
     def _handle_restart_service(self, command):
         service_id: int = command['number']
         print(f"Restart service {service_id}")

admin/admin_server.py

@@ -1,18 +1,3 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
 
 import os
 import signal

admin/auth.py

@@ -1,26 +1,9 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-
 import logging
 import uuid
 from functools import wraps
 from flask import request, jsonify
 
-from api.common.exceptions import AdminException
+from exceptions import AdminException
 from api.db.init_data import encode_to_base64
 from api.db.services import UserService
 

admin/config.py

@@ -1,20 +1,3 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-
 import logging
 import threading
 from enum import Enum
@@ -49,11 +32,9 @@ class BaseConfig(BaseModel):
     host: str
     port: int
     service_type: str
-    detail_func_name: str
 
     def to_dict(self) -> dict[str, Any]:
-        return {'id': self.id, 'name': self.name, 'host': self.host, 'port': self.port,
-                'service_type': self.service_type}
+        return {'id': self.id, 'name': self.name, 'host': self.host, 'port': self.port, 'service_type': self.service_type}
 
 
 class MetaConfig(BaseConfig):
@@ -228,8 +209,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
                 name: str = f'ragflow_{ragflow_count}'
                 host: str = v['host']
                 http_port: int = v['http_port']
-                config = RAGFlowServerConfig(id=id_count, name=name, host=host, port=http_port,
-                                             service_type="ragflow_server", detail_func_name="check_ragflow_server_alive")
+                config = RAGFlowServerConfig(id=id_count, name=name, host=host, port=http_port, service_type="ragflow_server")
                 configurations.append(config)
                 id_count += 1
             case "es":
@@ -242,8 +222,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
                 password: str = v.get('password')
                 config = ElasticsearchConfig(id=id_count, name=name, host=host, port=port, service_type="retrieval",
                                              retrieval_type="elasticsearch",
-                                             username=username, password=password,
-                                             detail_func_name="get_es_cluster_stats")
+                                             username=username, password=password)
                 configurations.append(config)
                 id_count += 1
 
@@ -255,7 +234,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
                 port = int(parts[1])
                 database: str = v.get('db_name', 'default_db')
                 config = InfinityConfig(id=id_count, name=name, host=host, port=port, service_type="retrieval", retrieval_type="infinity",
-                                        db_name=database, detail_func_name="get_infinity_status")
+                                        db_name=database)
                 configurations.append(config)
                 id_count += 1
             case "minio":
@@ -267,7 +246,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
                 user = v.get('user')
                 password = v.get('password')
                 config = MinioConfig(id=id_count, name=name, host=host, port=port, user=user, password=password, service_type="file_store",
-                                     store_type="minio", detail_func_name="check_minio_alive")
+                                     store_type="minio")
                 configurations.append(config)
                 id_count += 1
             case "redis":
@@ -279,7 +258,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
                 password = v.get('password')
                 db: int = v.get('db')
                 config = RedisConfig(id=id_count, name=name, host=host, port=port, password=password, database=db,
-                                     service_type="message_queue", mq_type="redis", detail_func_name="get_redis_info")
+                                     service_type="message_queue", mq_type="redis")
                 configurations.append(config)
                 id_count += 1
             case "mysql":
@@ -289,7 +268,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
                 username = v.get('user')
                 password = v.get('password')
                 config = MySQLConfig(id=id_count, name=name, host=host, port=port, username=username, password=password,
-                                     service_type="meta_data", meta_type="mysql", detail_func_name="get_mysql_status")
+                                     service_type="meta_data", meta_type="mysql")
                 configurations.append(config)
                 id_count += 1
             case "admin":

admin/models.py

@@ -1,15 +0,0 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#

admin/responses.py

@@ -1,34 +1,15 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-
 from flask import jsonify
 
-
-def success_response(data=None, message="Success", code=0):
+def success_response(data=None, message="Success", code = 0):
     return jsonify({
         "code": code,
         "message": message,
         "data": data
     }), 200
 
-
 def error_response(message="Error", code=-1, data=None):
     return jsonify({
         "code": code,
         "message": message,
         "data": data
-    }), 400
+    }), 400

admin/routes.py

@@ -1,26 +1,9 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-
 from flask import Blueprint, request
 
 from auth import login_verify
 from responses import success_response, error_response
 from services import UserMgr, ServiceMgr, UserServiceMgr
-from api.common.exceptions import AdminException
+from exceptions import AdminException
 
 admin_bp = Blueprint('admin', __name__, url_prefix='/api/v1/admin')
 
@@ -59,7 +42,7 @@ def create_user():
         res = UserMgr.create_user(username, password, role)
         if res["success"]:
             user_info = res["user_info"]
-            user_info.pop("password")  # do not return password
+            user_info.pop("password") # do not return password
             return success_response(user_info, "User created successfully")
         else:
             return error_response("create user failed")
@@ -119,7 +102,6 @@ def alter_user_activate_status(username):
     except Exception as e:
         return error_response(str(e), 500)
 
-
 @admin_bp.route('/users/<username>', methods=['GET'])
 @login_verify
 def get_user_details(username):
@@ -132,7 +114,6 @@ def get_user_details(username):
     except Exception as e:
         return error_response(str(e), 500)
 
-
 @admin_bp.route('/users/<username>/datasets', methods=['GET'])
 @login_verify
 def get_user_datasets(username):

admin/services.py

@@ -1,20 +1,3 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-
 import re
 from werkzeug.security import check_password_hash
 from api.db import ActiveEnum
@@ -24,20 +7,16 @@ from api.db.services.canvas_service import UserCanvasService
 from api.db.services.user_service import TenantService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.utils.crypt import decrypt
-from api.utils import health_utils
-
-from api.common.exceptions import AdminException, UserAlreadyExistsError, UserNotFoundError
+from exceptions import AdminException, UserAlreadyExistsError, UserNotFoundError
 from config import SERVICE_CONFIGS
 
-
 class UserMgr:
     @staticmethod
     def get_all_users():
         users = UserService.get_all_users()
         result = []
         for user in users:
-            result.append({'email': user.email, 'nickname': user.nickname, 'create_date': user.create_date,
-                           'is_active': user.is_active})
+            result.append({'email': user.email, 'nickname': user.nickname, 'create_date': user.create_date, 'is_active': user.is_active})
         return result
 
     @staticmethod
@@ -131,7 +110,6 @@ class UserMgr:
         UserService.update_user(usr.id, {"is_active": target_status})
         return f"Turn {_activate_status} user activate status successfully!"
 
-
 class UserServiceMgr:
 
     @staticmethod
@@ -170,7 +148,6 @@ class UserServiceMgr:
             'canvas_category': r['canvas_category']
         } for r in res]
 
-
 class ServiceMgr:
 
     @staticmethod
@@ -187,22 +164,7 @@ class ServiceMgr:
 
     @staticmethod
     def get_service_details(service_id: int):
-        service_id = int(service_id)
-        configs = SERVICE_CONFIGS.configs
-        service_config_mapping = {
-            c.id: {
-                'name': c.name,
-                'detail_func_name': c.detail_func_name
-            } for c in configs
-        }
-        service_info = service_config_mapping.get(service_id, {})
-        if not service_info:
-            raise AdminException(f"Invalid service_id: {service_id}")
-
-        detail_func = getattr(health_utils, service_info.get('detail_func_name'))
-        res = detail_func()
-        res.update({'service_name': service_info.get('name')})
-        return res
+        raise AdminException("get_service_details: not implemented")
 
     @staticmethod
     def shutdown_service(service_id: int):

agent/component/agent_with_tools.py

@@ -346,7 +346,3 @@ Respond immediately with your final comprehensive answer.
 
         return "Error occurred."
 
-    def reset(self):
-        for k, cpn in self.tools.items():
-            cpn.reset()
-

agent/component/invoke.py

@@ -19,12 +19,11 @@ import os
 import re
 import time
 from abc import ABC
-
 import requests
 
-from agent.component.base import ComponentBase, ComponentParamBase
 from api.utils.api_utils import timeout
 from deepdoc.parser import HtmlParser
+from agent.component.base import ComponentBase, ComponentParamBase
 
 
 class InvokeParam(ComponentParamBase):
@@ -44,11 +43,11 @@ class InvokeParam(ComponentParamBase):
         self.datatype = "json"  # New parameter to determine data posting type
 
     def check(self):
-        self.check_valid_value(self.method.lower(), "Type of content from the crawler", ["get", "post", "put"])
+        self.check_valid_value(self.method.lower(), "Type of content from the crawler", ['get', 'post', 'put'])
         self.check_empty(self.url, "End point URL")
         self.check_positive_integer(self.timeout, "Timeout time in second")
         self.check_boolean(self.clean_html, "Clean HTML")
-        self.check_valid_value(self.datatype.lower(), "Data post type", ["json", "formdata"])  # Check for valid datapost value
+        self.check_valid_value(self.datatype.lower(), "Data post type", ['json', 'formdata'])  # Check for valid datapost value
 
 
 class Invoke(ComponentBase, ABC):
@@ -64,18 +63,6 @@ class Invoke(ComponentBase, ABC):
                 args[para["key"]] = self._canvas.get_variable_value(para["ref"])
 
         url = self._param.url.strip()
-
-        def replace_variable(match):
-            var_name = match.group(1)
-            try:
-                value = self._canvas.get_variable_value(var_name)
-                return str(value or "")
-            except Exception:
-                return ""
-
-        # {base_url} or {component_id@variable_name}
-        url = re.sub(r"\{([a-zA-Z_][a-zA-Z0-9_.@-]*)\}", replace_variable, url)
-
         if url.find("http") != 0:
             url = "http://" + url
 
@@ -88,32 +75,52 @@ class Invoke(ComponentBase, ABC):
             proxies = {"http": self._param.proxy, "https": self._param.proxy}
 
         last_e = ""
-        for _ in range(self._param.max_retries + 1):
+        for _ in range(self._param.max_retries+1):
             try:
-                if method == "get":
-                    response = requests.get(url=url, params=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
+                if method == 'get':
+                    response = requests.get(url=url,
+                                            params=args,
+                                            headers=headers,
+                                            proxies=proxies,
+                                            timeout=self._param.timeout)
                     if self._param.clean_html:
                         sections = HtmlParser()(None, response.content)
                         self.set_output("result", "\n".join(sections))
                     else:
                         self.set_output("result", response.text)
 
-                if method == "put":
-                    if self._param.datatype.lower() == "json":
-                        response = requests.put(url=url, json=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
+                if method == 'put':
+                    if self._param.datatype.lower() == 'json':
+                        response = requests.put(url=url,
+                                                json=args,
+                                                headers=headers,
+                                                proxies=proxies,
+                                                timeout=self._param.timeout)
                     else:
-                        response = requests.put(url=url, data=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
+                        response = requests.put(url=url,
+                                                data=args,
+                                                headers=headers,
+                                                proxies=proxies,
+                                                timeout=self._param.timeout)
                     if self._param.clean_html:
                         sections = HtmlParser()(None, response.content)
                         self.set_output("result", "\n".join(sections))
                     else:
                         self.set_output("result", response.text)
 
-                if method == "post":
-                    if self._param.datatype.lower() == "json":
-                        response = requests.post(url=url, json=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
+                if method == 'post':
+                    if self._param.datatype.lower() == 'json':
+                        response = requests.post(url=url,
+                                                 json=args,
+                                                 headers=headers,
+                                                 proxies=proxies,
+                                                 timeout=self._param.timeout)
                     else:
-                        response = requests.post(url=url, data=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
+                        response = requests.post(url=url,
+                                                 data=args,
+                                                 headers=headers,
+                                                 proxies=proxies,
+                                                 timeout=self._param.timeout)
                     if self._param.clean_html:
                         self.set_output("result", "\n".join(sections))
                     else:

agent/tools/code_exec.py

@@ -156,8 +156,8 @@ class CodeExec(ToolBase, ABC):
             self.set_output("_ERROR", "construct code request error: " + str(e))
 
         try:
-            resp = requests.post(url=f"http://{settings.SANDBOX_HOST}:9385/run", json=code_req, timeout=int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
-            logging.info(f"http://{settings.SANDBOX_HOST}:9385/run,  code_req: {code_req}, resp.status_code {resp.status_code}:")
+            resp = requests.post(url=f"http://{settings.SANDBOX_HOST}:9385/run", json=code_req, timeout=os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))
+            logging.info(f"http://{settings.SANDBOX_HOST}:9385/run", code_req, resp.status_code)
             if resp.status_code != 200:
                 resp.raise_for_status()
             body = resp.json()

agent/tools/retrieval.py

@@ -121,7 +121,7 @@ class Retrieval(ToolBase, ABC):
 
         if kbs:
             query = re.sub(r"^user[:：\s]*", "", query, flags=re.IGNORECASE)
-            kbinfos = settings.retriever.retrieval(
+            kbinfos = settings.retrievaler.retrieval(
                 query,
                 embd_mdl,
                 [kb.tenant_id for kb in kbs],
@@ -135,7 +135,7 @@ class Retrieval(ToolBase, ABC):
                 rank_feature=label_question(query, kbs),
             )
             if self._param.use_kg:
-                ck = settings.kg_retriever.retrieval(query,
+                ck = settings.kg_retrievaler.retrieval(query,
                                                        [kb.tenant_id for kb in kbs],
                                                        kb_ids,
                                                        embd_mdl,
@@ -146,7 +146,7 @@ class Retrieval(ToolBase, ABC):
             kbinfos = {"chunks": [], "doc_aggs": []}
 
         if self._param.use_kg and kbs:
-            ck = settings.kg_retriever.retrieval(query, [kb.tenant_id for kb in kbs], filtered_kb_ids, embd_mdl, LLMBundle(kbs[0].tenant_id, LLMType.CHAT))
+            ck = settings.kg_retrievaler.retrieval(query, [kb.tenant_id for kb in kbs], filtered_kb_ids, embd_mdl, LLMBundle(kbs[0].tenant_id, LLMType.CHAT))
             if ck["content_with_weight"]:
                 ck["content"] = ck["content_with_weight"]
                 del ck["content_with_weight"]

agent/tools/searxng.py

@@ -85,7 +85,7 @@ class SearXNG(ToolBase, ABC):
             self.set_output("formalized_content", "")
             return ""
 
-        searxng_url = (getattr(self._param, "searxng_url", "") or kwargs.get("searxng_url") or "").strip()
+        searxng_url = (kwargs.get("searxng_url") or getattr(self._param, "searxng_url", "") or "").strip()
         # In try-run, if no URL configured, just return empty instead of raising
         if not searxng_url:
             self.set_output("formalized_content", "")

api/apps/api_app.py

@@ -536,7 +536,7 @@ def list_chunks():
             )
         kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
 
-        res = settings.retriever.chunk_list(doc_id, tenant_id, kb_ids)
+        res = settings.retrievaler.chunk_list(doc_id, tenant_id, kb_ids)
         res = [
             {
                 "content": res_item["content_with_weight"],
@@ -884,7 +884,7 @@ def retrieval():
         if req.get("keyword", False):
             chat_mdl = LLMBundle(kbs[0].tenant_id, LLMType.CHAT)
             question += keyword_extraction(chat_mdl, question)
-        ranks = settings.retriever.retrieval(question, embd_mdl, kbs[0].tenant_id, kb_ids, page, size,
+        ranks = settings.retrievaler.retrieval(question, embd_mdl, kbs[0].tenant_id, kb_ids, page, size,
                                                similarity_threshold, vector_similarity_weight, top,
                                                doc_ids, rerank_mdl=rerank_mdl, highlight= highlight,
                                                rank_feature=label_question(question, kbs))

api/apps/chunk_app.py

@@ -60,7 +60,7 @@ def list_chunk():
         }
         if "available_int" in req:
             query["available_int"] = int(req["available_int"])
-        sres = settings.retriever.search(query, search.index_name(tenant_id), kb_ids, highlight=True)
+        sres = settings.retrievaler.search(query, search.index_name(tenant_id), kb_ids, highlight=True)
         res = {"total": sres.total, "chunks": [], "doc": doc.to_dict()}
         for id in sres.ids:
             d = {
@@ -346,7 +346,7 @@ def retrieval_test():
             question += keyword_extraction(chat_mdl, question)
 
         labels = label_question(question, [kb])
-        ranks = settings.retriever.retrieval(question, embd_mdl, tenant_ids, kb_ids, page, size,
+        ranks = settings.retrievaler.retrieval(question, embd_mdl, tenant_ids, kb_ids, page, size,
                                float(req.get("similarity_threshold", 0.0)),
                                float(req.get("vector_similarity_weight", 0.3)),
                                top,
@@ -354,7 +354,7 @@ def retrieval_test():
                                rank_feature=labels
                                )
         if use_kg:
-            ck = settings.kg_retriever.retrieval(question,
+            ck = settings.kg_retrievaler.retrieval(question,
                                                    tenant_ids,
                                                    kb_ids,
                                                    embd_mdl,
@@ -384,7 +384,7 @@ def knowledge_graph():
         "doc_ids": [doc_id],
         "knowledge_graph_kwd": ["graph", "mind_map"]
     }
-    sres = settings.retriever.search(req, search.index_name(tenant_id), kb_ids)
+    sres = settings.retrievaler.search(req, search.index_name(tenant_id), kb_ids)
     obj = {"graph": {}, "mind_map": {}}
     for id in sres.ids[:2]:
         ty = sres.field[id]["knowledge_graph_kwd"]

api/apps/document_app.py

@@ -24,7 +24,6 @@ from flask import request
 from flask_login import current_user, login_required
 
 from api import settings
-from api.common.check_team_permission import check_kb_team_permission
 from api.constants import FILE_NAME_LEN_LIMIT, IMG_BASE64_PREFIX
 from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileSource, FileType, ParserType, TaskStatus
 from api.db.db_models import File, Task
@@ -69,10 +68,8 @@ def upload():
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
-    if not check_kb_team_permission(kb, current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
-
     err, files = FileService.upload_document(kb, file_objs, current_user.id)
+
     if err:
         return get_json_result(data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
 
@@ -97,8 +94,6 @@ def web_crawl():
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
-    if check_kb_team_permission(kb, current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
 
     blob = html2pdf(url)
     if not blob:
@@ -557,8 +552,8 @@ def get(doc_id):
 @login_required
 @validate_request("doc_id")
 def change_parser():
-
     req = request.json
+
     if not DocumentService.accessible(req["doc_id"], current_user.id):
         return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
 
@@ -582,7 +577,7 @@ def change_parser():
                 settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id)
 
     try:
-        if "pipeline_id" in req and req["pipeline_id"] != "":
+        if "pipeline_id" in req:
             if doc.pipeline_id == req["pipeline_id"]:
                 return get_json_result(data=True)
             DocumentService.update_by_id(doc.id, {"pipeline_id": req["pipeline_id"]})

api/apps/file_app.py

@@ -21,7 +21,6 @@ import flask
 from flask import request
 from flask_login import login_required, current_user
 
-from api.common.check_team_permission import check_file_team_permission
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
@@ -247,7 +246,7 @@ def rm():
                 return get_data_error_result(message="File or Folder not found!")
             if not file.tenant_id:
                 return get_data_error_result(message="Tenant not found!")
-            if not check_file_team_permission(file, current_user.id):
+            if file.tenant_id != current_user.id:
                 return get_json_result(data=False, message='No authorization.', code=settings.RetCode.AUTHENTICATION_ERROR)
             if file.source_type == FileSource.KNOWLEDGEBASE:
                 continue
@@ -295,7 +294,7 @@ def rename():
         e, file = FileService.get_by_id(req["file_id"])
         if not e:
             return get_data_error_result(message="File not found!")
-        if not check_file_team_permission(file, current_user.id):
+        if file.tenant_id != current_user.id:
             return get_json_result(data=False, message='No authorization.', code=settings.RetCode.AUTHENTICATION_ERROR)
         if file.type != FileType.FOLDER.value \
             and pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
@@ -333,7 +332,7 @@ def get(file_id):
         e, file = FileService.get_by_id(file_id)
         if not e:
             return get_data_error_result(message="Document not found!")
-        if not check_file_team_permission(file, current_user.id):
+        if file.tenant_id != current_user.id:
             return get_json_result(data=False, message='No authorization.', code=settings.RetCode.AUTHENTICATION_ERROR)
 
         blob = STORAGE_IMPL.get(file.parent_id, file.location)
@@ -374,7 +373,7 @@ def move():
                 return get_data_error_result(message="File or Folder not found!")
             if not file.tenant_id:
                 return get_data_error_result(message="Tenant not found!")
-            if not check_file_team_permission(file, current_user.id):
+            if file.tenant_id != current_user.id:
                 return get_json_result(data=False, message='No authorization.', code=settings.RetCode.AUTHENTICATION_ERROR)
         fe, _ = FileService.get_by_id(parent_id)
         if not fe:

api/apps/kb_app.py

@@ -38,7 +38,6 @@ from api.constants import DATASET_NAME_LIMIT
 from rag.settings import PAGERANK_FLD
 from rag.utils.storage_factory import STORAGE_IMPL
 
-
 @manager.route('/create', methods=['post'])  # noqa: F821
 @login_required
 @validate_request("name")
@@ -282,7 +281,7 @@ def list_tags(kb_id):
     tenants = UserTenantService.get_tenants_by_user_id(current_user.id)
     tags = []
     for tenant in tenants:
-        tags += settings.retriever.all_tags(tenant["tenant_id"], [kb_id])
+        tags += settings.retrievaler.all_tags(tenant["tenant_id"], [kb_id])
     return get_json_result(data=tags)
 
 
@@ -301,7 +300,7 @@ def list_tags_from_kbs():
     tenants = UserTenantService.get_tenants_by_user_id(current_user.id)
     tags = []
     for tenant in tenants:
-        tags += settings.retriever.all_tags(tenant["tenant_id"], kb_ids)
+        tags += settings.retrievaler.all_tags(tenant["tenant_id"], kb_ids)
     return get_json_result(data=tags)
 
 
@@ -362,7 +361,7 @@ def knowledge_graph(kb_id):
     obj = {"graph": {}, "mind_map": {}}
     if not settings.docStoreConn.indexExist(search.index_name(kb.tenant_id), kb_id):
         return get_json_result(data=obj)
-    sres = settings.retriever.search(req, search.index_name(kb.tenant_id), [kb_id])
+    sres = settings.retrievaler.search(req, search.index_name(kb.tenant_id), [kb_id])
     if not len(sres.ids):
         return get_json_result(data=obj)
 

api/apps/plugin_app.py

@@ -1,26 +1,8 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-
 from flask import Response
 from flask_login import login_required
 from api.utils.api_utils import get_json_result
 from plugin import GlobalPluginManager
 
-
 @manager.route('/llm_tools', methods=['GET'])  # noqa: F821
 @login_required
 def llm_tools() -> Response:

api/apps/sdk/agent.py

@@ -25,7 +25,6 @@ from api.utils.api_utils import get_data_error_result, get_error_data_result, ge
 from api.utils.api_utils import get_result
 from flask import request
 
-
 @manager.route('/agents', methods=['GET'])  # noqa: F821
 @token_required
 def list_agents(tenant_id):
@@ -42,7 +41,7 @@ def list_agents(tenant_id):
         desc = False
     else:
         desc = True
-    canvas = UserCanvasService.get_list(tenant_id, page_number, items_per_page, orderby, desc, id, title)
+    canvas = UserCanvasService.get_list(tenant_id,page_number,items_per_page,orderby,desc,id,title)
     return get_result(data=canvas)
 
 
@@ -94,7 +93,7 @@ def update_agent(tenant_id: str, agent_id: str):
             req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
 
         req["dsl"] = json.loads(req["dsl"])
-
+    
     if req.get("title") is not None:
         req["title"] = req["title"].strip()
 

api/apps/sdk/dataset.py

@@ -215,8 +215,7 @@ def delete(tenant_id):
                     continue
                 kb_id_instance_pairs.append((kb_id, kb))
             if len(error_kb_ids) > 0:
-                return get_error_permission_result(
-                    message=f"""User '{tenant_id}' lacks permission for datasets: '{", ".join(error_kb_ids)}'""")
+                return get_error_permission_result(message=f"""User '{tenant_id}' lacks permission for datasets: '{", ".join(error_kb_ids)}'""")
 
         errors = []
         success_count = 0
@@ -233,8 +232,7 @@ def delete(tenant_id):
                     ]
                 )
                 File2DocumentService.delete_by_document_id(doc.id)
-            FileService.filter_delete(
-                [File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kb.name])
+            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kb.name])
             if not KnowledgebaseService.delete_by_id(kb_id):
                 errors.append(f"Delete dataset error for {kb_id}")
                 continue
@@ -331,8 +329,7 @@ def update(tenant_id, dataset_id):
     try:
         kb = KnowledgebaseService.get_or_none(id=dataset_id, tenant_id=tenant_id)
         if kb is None:
-            return get_error_permission_result(
-                message=f"User '{tenant_id}' lacks permission for dataset '{dataset_id}'")
+            return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{dataset_id}'")
 
         if req.get("parser_config"):
             req["parser_config"] = deep_merge(kb.parser_config, req["parser_config"])
@@ -344,8 +341,7 @@ def update(tenant_id, dataset_id):
             del req["parser_config"]
 
         if "name" in req and req["name"].lower() != kb.name.lower():
-            exists = KnowledgebaseService.get_or_none(name=req["name"], tenant_id=tenant_id,
-                                                      status=StatusEnum.VALID.value)
+            exists = KnowledgebaseService.get_or_none(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)
             if exists:
                 return get_error_data_result(message=f"Dataset name '{req['name']}' already exists")
 
@@ -353,8 +349,7 @@ def update(tenant_id, dataset_id):
             if not req["embd_id"]:
                 req["embd_id"] = kb.embd_id
             if kb.chunk_num != 0 and req["embd_id"] != kb.embd_id:
-                return get_error_data_result(
-                    message=f"When chunk_num ({kb.chunk_num}) > 0, embedding_model must remain {kb.embd_id}")
+                return get_error_data_result(message=f"When chunk_num ({kb.chunk_num}) > 0, embedding_model must remain {kb.embd_id}")
             ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
             if not ok:
                 return err
@@ -364,12 +359,10 @@ def update(tenant_id, dataset_id):
                 return get_error_argument_result(message="'pagerank' can only be set when doc_engine is elasticsearch")
 
             if req["pagerank"] > 0:
-                settings.docStoreConn.update({"kb_id": kb.id}, {PAGERANK_FLD: req["pagerank"]},
-                                             search.index_name(kb.tenant_id), kb.id)
+                settings.docStoreConn.update({"kb_id": kb.id}, {PAGERANK_FLD: req["pagerank"]}, search.index_name(kb.tenant_id), kb.id)
             else:
                 # Elasticsearch requires PAGERANK_FLD be non-zero!
-                settings.docStoreConn.update({"exists": PAGERANK_FLD}, {"remove": PAGERANK_FLD},
-                                             search.index_name(kb.tenant_id), kb.id)
+                settings.docStoreConn.update({"exists": PAGERANK_FLD}, {"remove": PAGERANK_FLD}, search.index_name(kb.tenant_id), kb.id)
 
         if not KnowledgebaseService.update_by_id(kb.id, req):
             return get_error_data_result(message="Update dataset error.(Database error)")
@@ -461,7 +454,7 @@ def list_datasets(tenant_id):
                 return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{name}'")
 
         tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)
-        kbs, total = KnowledgebaseService.get_list(
+        kbs = KnowledgebaseService.get_list(
             [m["tenant_id"] for m in tenants],
             tenant_id,
             args["page"],
@@ -475,15 +468,14 @@ def list_datasets(tenant_id):
         response_data_list = []
         for kb in kbs:
             response_data_list.append(remap_dictionary_keys(kb))
-        return get_result(data=response_data_list, total=total)
+        return get_result(data=response_data_list)
     except OperationalError as e:
         logging.exception(e)
         return get_error_data_result(message="Database operation failed")
 
-
 @manager.route('/datasets/<dataset_id>/knowledge_graph', methods=['GET'])  # noqa: F821
 @token_required
-def knowledge_graph(tenant_id, dataset_id):
+def knowledge_graph(tenant_id,dataset_id):
     if not KnowledgebaseService.accessible(dataset_id, tenant_id):
         return get_result(
             data=False,
@@ -499,7 +491,7 @@ def knowledge_graph(tenant_id, dataset_id):
     obj = {"graph": {}, "mind_map": {}}
     if not settings.docStoreConn.indexExist(search.index_name(kb.tenant_id), dataset_id):
         return get_result(data=obj)
-    sres = settings.retriever.search(req, search.index_name(kb.tenant_id), [dataset_id])
+    sres = settings.retrievaler.search(req, search.index_name(kb.tenant_id), [dataset_id])
     if not len(sres.ids):
         return get_result(data=obj)
 
@@ -515,16 +507,14 @@ def knowledge_graph(tenant_id, dataset_id):
     if "nodes" in obj["graph"]:
         obj["graph"]["nodes"] = sorted(obj["graph"]["nodes"], key=lambda x: x.get("pagerank", 0), reverse=True)[:256]
         if "edges" in obj["graph"]:
-            node_id_set = {o["id"] for o in obj["graph"]["nodes"]}
-            filtered_edges = [o for o in obj["graph"]["edges"] if
-                              o["source"] != o["target"] and o["source"] in node_id_set and o["target"] in node_id_set]
+            node_id_set = { o["id"] for o in obj["graph"]["nodes"] }
+            filtered_edges = [o for o in obj["graph"]["edges"] if o["source"] != o["target"] and o["source"] in node_id_set and o["target"] in node_id_set]
             obj["graph"]["edges"] = sorted(filtered_edges, key=lambda x: x.get("weight", 0), reverse=True)[:128]
     return get_result(data=obj)
 
-
 @manager.route('/datasets/<dataset_id>/knowledge_graph', methods=['DELETE'])  # noqa: F821
 @token_required
-def delete_knowledge_graph(tenant_id, dataset_id):
+def delete_knowledge_graph(tenant_id,dataset_id):
     if not KnowledgebaseService.accessible(dataset_id, tenant_id):
         return get_result(
             data=False,
@@ -532,7 +522,6 @@ def delete_knowledge_graph(tenant_id, dataset_id):
             code=settings.RetCode.AUTHENTICATION_ERROR
         )
     _, kb = KnowledgebaseService.get_by_id(dataset_id)
-    settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]},
-                                 search.index_name(kb.tenant_id), dataset_id)
+    settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), dataset_id)
 
     return get_result(data=True)

api/apps/sdk/dify_retrieval.py

@@ -1,4 +1,4 @@
-#
+    #
 #  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -38,9 +38,9 @@ def retrieval(tenant_id):
     retrieval_setting = req.get("retrieval_setting", {})
     similarity_threshold = float(retrieval_setting.get("score_threshold", 0.0))
     top = int(retrieval_setting.get("top_k", 1024))
-    metadata_condition = req.get("metadata_condition", {})
+    metadata_condition = req.get("metadata_condition",{})
     metas = DocumentService.get_meta_by_kbs([kb_id])
-
+ 
     doc_ids = []
     try:
 
@@ -50,12 +50,12 @@ def retrieval(tenant_id):
 
         embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
         print(metadata_condition)
-        # print("after", convert_conditions(metadata_condition))
+        print("after",convert_conditions(metadata_condition))
         doc_ids.extend(meta_filter(metas, convert_conditions(metadata_condition)))
-        # print("doc_ids", doc_ids)
+        print("doc_ids",doc_ids)
         if not doc_ids and metadata_condition is not None:
             doc_ids = ['-999']
-        ranks = settings.retriever.retrieval(
+        ranks = settings.retrievaler.retrieval(
             question,
             embd_mdl,
             kb.tenant_id,
@@ -70,17 +70,17 @@ def retrieval(tenant_id):
         )
 
         if use_kg:
-            ck = settings.kg_retriever.retrieval(question,
-                                                 [tenant_id],
-                                                 [kb_id],
-                                                 embd_mdl,
-                                                 LLMBundle(kb.tenant_id, LLMType.CHAT))
+            ck = settings.kg_retrievaler.retrieval(question,
+                                                   [tenant_id],
+                                                   [kb_id],
+                                                   embd_mdl,
+                                                   LLMBundle(kb.tenant_id, LLMType.CHAT))
             if ck["content_with_weight"]:
                 ranks["chunks"].insert(0, ck)
 
         records = []
         for c in ranks["chunks"]:
-            e, doc = DocumentService.get_by_id(c["doc_id"])
+            e, doc = DocumentService.get_by_id( c["doc_id"])
             c.pop("vector", None)
             meta = getattr(doc, 'meta_fields', {})
             meta["doc_id"] = c["doc_id"]
@@ -100,3 +100,5 @@ def retrieval(tenant_id):
             )
         logging.exception(e)
         return build_error_result(message=str(e), code=settings.RetCode.SERVER_ERROR)
+
+

api/apps/sdk/doc.py

@@ -982,7 +982,7 @@ def list_chunks(tenant_id, dataset_id, document_id):
         _ = Chunk(**final_chunk)
 
     elif settings.docStoreConn.indexExist(search.index_name(tenant_id), dataset_id):
-        sres = settings.retriever.search(query, search.index_name(tenant_id), [dataset_id], emb_mdl=None, highlight=True)
+        sres = settings.retrievaler.search(query, search.index_name(tenant_id), [dataset_id], emb_mdl=None, highlight=True)
         res["total"] = sres.total
         for id in sres.ids:
             d = {
@@ -1446,7 +1446,7 @@ def retrieval_test(tenant_id):
             chat_mdl = LLMBundle(kb.tenant_id, LLMType.CHAT)
             question += keyword_extraction(chat_mdl, question)
 
-        ranks = settings.retriever.retrieval(
+        ranks = settings.retrievaler.retrieval(
             question,
             embd_mdl,
             tenant_ids,
@@ -1462,7 +1462,7 @@ def retrieval_test(tenant_id):
             rank_feature=label_question(question, kbs),
         )
         if use_kg:
-            ck = settings.kg_retriever.retrieval(question, [k.tenant_id for k in kbs], kb_ids, embd_mdl, LLMBundle(kb.tenant_id, LLMType.CHAT))
+            ck = settings.kg_retrievaler.retrieval(question, [k.tenant_id for k in kbs], kb_ids, embd_mdl, LLMBundle(kb.tenant_id, LLMType.CHAT))
             if ck["content_with_weight"]:
                 ranks["chunks"].insert(0, ck)
 

api/apps/sdk/files.py

@@ -1,20 +1,3 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-
 import pathlib
 import re
 
@@ -34,8 +17,7 @@ from api.utils.api_utils import get_json_result
 from api.utils.file_utils import filename_type
 from rag.utils.storage_factory import STORAGE_IMPL
 
-
-@manager.route('/file/upload', methods=['POST'])  # noqa: F821
+@manager.route('/file/upload', methods=['POST']) # noqa: F821
 @token_required
 def upload(tenant_id):
     """
@@ -115,14 +97,12 @@ def upload(tenant_id):
                 e, file = FileService.get_by_id(file_id_list[len_id_list - 1])
                 if not e:
                     return get_json_result(data=False, message="Folder not found!", code=404)
-                last_folder = FileService.create_folder(file, file_id_list[len_id_list - 1], file_obj_names,
-                                                        len_id_list)
+                last_folder = FileService.create_folder(file, file_id_list[len_id_list - 1], file_obj_names, len_id_list)
             else:
                 e, file = FileService.get_by_id(file_id_list[len_id_list - 2])
                 if not e:
                     return get_json_result(data=False, message="Folder not found!", code=404)
-                last_folder = FileService.create_folder(file, file_id_list[len_id_list - 2], file_obj_names,
-                                                        len_id_list)
+                last_folder = FileService.create_folder(file, file_id_list[len_id_list - 2], file_obj_names, len_id_list)
 
             filetype = filename_type(file_obj_names[file_len - 1])
             location = file_obj_names[file_len - 1]
@@ -149,7 +129,7 @@ def upload(tenant_id):
         return server_error_response(e)
 
 
-@manager.route('/file/create', methods=['POST'])  # noqa: F821
+@manager.route('/file/create', methods=['POST']) # noqa: F821
 @token_required
 def create(tenant_id):
     """
@@ -227,7 +207,7 @@ def create(tenant_id):
         return server_error_response(e)
 
 
-@manager.route('/file/list', methods=['GET'])  # noqa: F821
+@manager.route('/file/list', methods=['GET']) # noqa: F821
 @token_required
 def list_files(tenant_id):
     """
@@ -319,7 +299,7 @@ def list_files(tenant_id):
         return server_error_response(e)
 
 
-@manager.route('/file/root_folder', methods=['GET'])  # noqa: F821
+@manager.route('/file/root_folder', methods=['GET']) # noqa: F821
 @token_required
 def get_root_folder(tenant_id):
     """
@@ -355,7 +335,7 @@ def get_root_folder(tenant_id):
         return server_error_response(e)
 
 
-@manager.route('/file/parent_folder', methods=['GET'])  # noqa: F821
+@manager.route('/file/parent_folder', methods=['GET']) # noqa: F821
 @token_required
 def get_parent_folder():
     """
@@ -400,7 +380,7 @@ def get_parent_folder():
         return server_error_response(e)
 
 
-@manager.route('/file/all_parent_folder', methods=['GET'])  # noqa: F821
+@manager.route('/file/all_parent_folder', methods=['GET']) # noqa: F821
 @token_required
 def get_all_parent_folders(tenant_id):
     """
@@ -448,7 +428,7 @@ def get_all_parent_folders(tenant_id):
         return server_error_response(e)
 
 
-@manager.route('/file/rm', methods=['POST'])  # noqa: F821
+@manager.route('/file/rm', methods=['POST']) # noqa: F821
 @token_required
 def rm(tenant_id):
     """
@@ -522,7 +502,7 @@ def rm(tenant_id):
         return server_error_response(e)
 
 
-@manager.route('/file/rename', methods=['POST'])  # noqa: F821
+@manager.route('/file/rename', methods=['POST']) # noqa: F821
 @token_required
 def rename(tenant_id):
     """
@@ -562,8 +542,7 @@ def rename(tenant_id):
         if not e:
             return get_json_result(message="File not found!", code=404)
 
-        if file.type != FileType.FOLDER.value and pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
-                file.name.lower()).suffix:
+        if file.type != FileType.FOLDER.value and pathlib.Path(req["name"].lower()).suffix != pathlib.Path(file.name.lower()).suffix:
             return get_json_result(data=False, message="The extension of file can't be changed", code=400)
 
         for existing_file in FileService.query(name=req["name"], pf_id=file.parent_id):
@@ -583,9 +562,9 @@ def rename(tenant_id):
         return server_error_response(e)
 
 
-@manager.route('/file/get/<file_id>', methods=['GET'])  # noqa: F821
+@manager.route('/file/get/<file_id>', methods=['GET']) # noqa: F821
 @token_required
-def get(tenant_id, file_id):
+def get(tenant_id,file_id):
     """
     Download a file.
     ---
@@ -631,7 +610,7 @@ def get(tenant_id, file_id):
         return server_error_response(e)
 
 
-@manager.route('/file/mv', methods=['POST'])  # noqa: F821
+@manager.route('/file/mv', methods=['POST']) # noqa: F821
 @token_required
 def move(tenant_id):
     """
@@ -690,7 +669,6 @@ def move(tenant_id):
     except Exception as e:
         return server_error_response(e)
 
-
 @manager.route('/file/convert', methods=['POST'])  # noqa: F821
 @token_required
 def convert(tenant_id):
@@ -757,4 +735,4 @@ def convert(tenant_id):
                     file2documents.append(file2document.to_json())
         return get_json_result(data=file2documents)
     except Exception as e:
-        return server_error_response(e)
+        return server_error_response(e)

api/apps/sdk/session.py

@@ -36,8 +36,7 @@ from api.db.services.llm_service import LLMBundle
 from api.db.services.search_service import SearchService
 from api.db.services.user_service import UserTenantService
 from api.utils import get_uuid
-from api.utils.api_utils import check_duplicate_ids, get_data_openai, get_error_data_result, get_json_result, \
-    get_result, server_error_response, token_required, validate_request
+from api.utils.api_utils import check_duplicate_ids, get_data_openai, get_error_data_result, get_json_result, get_result, server_error_response, token_required, validate_request
 from rag.app.tag import label_question
 from rag.prompts.template import load_prompt
 from rag.prompts.generator import cross_languages, gen_meta_filter, keyword_extraction, chunks_format
@@ -89,8 +88,7 @@ def create_agent_session(tenant_id, agent_id):
     canvas.reset()
 
     cvs.dsl = json.loads(str(canvas))
-    conv = {"id": session_id, "dialog_id": cvs.id, "user_id": user_id,
-            "message": [{"role": "assistant", "content": canvas.get_prologue()}], "source": "agent", "dsl": cvs.dsl}
+    conv = {"id": session_id, "dialog_id": cvs.id, "user_id": user_id, "message": [{"role": "assistant", "content": canvas.get_prologue()}], "source": "agent", "dsl": cvs.dsl}
     API4ConversationService.save(**conv)
     conv["agent_id"] = conv.pop("dialog_id")
     return get_result(data=conv)
@@ -281,7 +279,7 @@ def chat_completion_openai_like(tenant_id, chat_id):
                     reasoning_match = re.search(r"<think>(.*?)</think>", answer, flags=re.DOTALL)
                     if reasoning_match:
                         reasoning_part = reasoning_match.group(1)
-                        content_part = answer[reasoning_match.end():]
+                        content_part = answer[reasoning_match.end() :]
                     else:
                         reasoning_part = ""
                         content_part = answer
@@ -326,8 +324,7 @@ def chat_completion_openai_like(tenant_id, chat_id):
             response["choices"][0]["delta"]["content"] = None
             response["choices"][0]["delta"]["reasoning_content"] = None
             response["choices"][0]["finish_reason"] = "stop"
-            response["usage"] = {"prompt_tokens": len(prompt), "completion_tokens": token_used,
-                                 "total_tokens": len(prompt) + token_used}
+            response["usage"] = {"prompt_tokens": len(prompt), "completion_tokens": token_used, "total_tokens": len(prompt) + token_used}
             if need_reference:
                 response["choices"][0]["delta"]["reference"] = chunks_format(last_ans.get("reference", []))
                 response["choices"][0]["delta"]["final_content"] = last_ans.get("answer", "")
@@ -562,8 +559,7 @@ def list_agent_session(tenant_id, agent_id):
         desc = True
     # dsl defaults to True in all cases except for False and false
     include_dsl = request.args.get("dsl") != "False" and request.args.get("dsl") != "false"
-    total, convs = API4ConversationService.get_list(agent_id, tenant_id, page_number, items_per_page, orderby, desc, id,
-                                                    user_id, include_dsl)
+    total, convs = API4ConversationService.get_list(agent_id, tenant_id, page_number, items_per_page, orderby, desc, id, user_id, include_dsl)
     if not convs:
         return get_result(data=[])
     for conv in convs:
@@ -585,8 +581,7 @@ def list_agent_session(tenant_id, agent_id):
                 if message_num != 0 and messages[message_num]["role"] != "user":
                     chunk_list = []
                     # Add boundary and type checks to prevent KeyError
-                    if chunk_num < len(conv["reference"]) and conv["reference"][chunk_num] is not None and isinstance(
-                            conv["reference"][chunk_num], dict) and "chunks" in conv["reference"][chunk_num]:
+                    if chunk_num < len(conv["reference"]) and conv["reference"][chunk_num] is not None and isinstance(conv["reference"][chunk_num], dict) and "chunks" in conv["reference"][chunk_num]:
                         chunks = conv["reference"][chunk_num]["chunks"]
                         for chunk in chunks:
                             # Ensure chunk is a dictionary before calling get method
@@ -644,16 +639,13 @@ def delete(tenant_id, chat_id):
 
     if errors:
         if success_count > 0:
-            return get_result(data={"success_count": success_count, "errors": errors},
-                              message=f"Partially deleted {success_count} sessions with {len(errors)} errors")
+            return get_result(data={"success_count": success_count, "errors": errors}, message=f"Partially deleted {success_count} sessions with {len(errors)} errors")
         else:
             return get_error_data_result(message="; ".join(errors))
 
     if duplicate_messages:
         if success_count > 0:
-            return get_result(
-                message=f"Partially deleted {success_count} sessions with {len(duplicate_messages)} errors",
-                data={"success_count": success_count, "errors": duplicate_messages})
+            return get_result(message=f"Partially deleted {success_count} sessions with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages})
         else:
             return get_error_data_result(message=";".join(duplicate_messages))
 
@@ -699,16 +691,13 @@ def delete_agent_session(tenant_id, agent_id):
 
     if errors:
         if success_count > 0:
-            return get_result(data={"success_count": success_count, "errors": errors},
-                              message=f"Partially deleted {success_count} sessions with {len(errors)} errors")
+            return get_result(data={"success_count": success_count, "errors": errors}, message=f"Partially deleted {success_count} sessions with {len(errors)} errors")
         else:
             return get_error_data_result(message="; ".join(errors))
 
     if duplicate_messages:
         if success_count > 0:
-            return get_result(
-                message=f"Partially deleted {success_count} sessions with {len(duplicate_messages)} errors",
-                data={"success_count": success_count, "errors": duplicate_messages})
+            return get_result(message=f"Partially deleted {success_count} sessions with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages})
         else:
             return get_error_data_result(message=";".join(duplicate_messages))
 
@@ -741,9 +730,7 @@ def ask_about(tenant_id):
             for ans in ask(req["question"], req["kb_ids"], uid):
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
         except Exception as e:
-            yield "data:" + json.dumps(
-                {"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
-                ensure_ascii=False) + "\n\n"
+            yield "data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}}, ensure_ascii=False) + "\n\n"
         yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
 
     resp = Response(stream(), mimetype="text/event-stream")
@@ -895,9 +882,7 @@ def begin_inputs(agent_id):
         return get_error_data_result(f"Can't find agent by ID: {agent_id}")
 
     canvas = Canvas(json.dumps(cvs.dsl), objs[0].tenant_id)
-    return get_result(
-        data={"title": cvs.title, "avatar": cvs.avatar, "inputs": canvas.get_component_input_form("begin"),
-              "prologue": canvas.get_prologue(), "mode": canvas.get_mode()})
+    return get_result(data={"title": cvs.title, "avatar": cvs.avatar, "inputs": canvas.get_component_input_form("begin"), "prologue": canvas.get_prologue(), "mode": canvas.get_mode()})
 
 
 @manager.route("/searchbots/ask", methods=["POST"])  # noqa: F821
@@ -926,9 +911,7 @@ def ask_about_embedded():
             for ans in ask(req["question"], req["kb_ids"], uid, search_config=search_config):
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
         except Exception as e:
-            yield "data:" + json.dumps(
-                {"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
-                ensure_ascii=False) + "\n\n"
+            yield "data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}}, ensure_ascii=False) + "\n\n"
         yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
 
     resp = Response(stream(), mimetype="text/event-stream")
@@ -995,8 +978,7 @@ def retrieval_test_embedded():
                     tenant_ids.append(tenant.tenant_id)
                     break
             else:
-                return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.",
-                                       code=settings.RetCode.OPERATING_ERROR)
+                return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
 
         e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
         if not e:
@@ -1016,13 +998,11 @@ def retrieval_test_embedded():
             question += keyword_extraction(chat_mdl, question)
 
         labels = label_question(question, [kb])
-        ranks = settings.retriever.retrieval(
-            question, embd_mdl, tenant_ids, kb_ids, page, size, similarity_threshold, vector_similarity_weight, top,
-            doc_ids, rerank_mdl=rerank_mdl, highlight=req.get("highlight"), rank_feature=labels
+        ranks = settings.retrievaler.retrieval(
+            question, embd_mdl, tenant_ids, kb_ids, page, size, similarity_threshold, vector_similarity_weight, top, doc_ids, rerank_mdl=rerank_mdl, highlight=req.get("highlight"), rank_feature=labels
         )
         if use_kg:
-            ck = settings.kg_retriever.retrieval(question, tenant_ids, kb_ids, embd_mdl,
-                                                 LLMBundle(kb.tenant_id, LLMType.CHAT))
+            ck = settings.kg_retrievaler.retrieval(question, tenant_ids, kb_ids, embd_mdl, LLMBundle(kb.tenant_id, LLMType.CHAT))
             if ck["content_with_weight"]:
                 ranks["chunks"].insert(0, ck)
 
@@ -1033,8 +1013,7 @@ def retrieval_test_embedded():
         return get_json_result(data=ranks)
     except Exception as e:
         if str(e).find("not_found") > 0:
-            return get_json_result(data=False, message="No chunk found! Check the chunk status please!",
-                                   code=settings.RetCode.DATA_ERROR)
+            return get_json_result(data=False, message="No chunk found! Check the chunk status please!", code=settings.RetCode.DATA_ERROR)
         return server_error_response(e)
 
 
@@ -1103,8 +1082,7 @@ def detail_share_embedded():
             if SearchService.query(tenant_id=tenant.tenant_id, id=search_id):
                 break
         else:
-            return get_json_result(data=False, message="Has no permission for this operation.",
-                                   code=settings.RetCode.OPERATING_ERROR)
+            return get_json_result(data=False, message="Has no permission for this operation.", code=settings.RetCode.OPERATING_ERROR)
 
         search = SearchService.get_detail(search_id)
         if not search:

api/apps/system_app.py

@@ -162,7 +162,7 @@ def status():
         task_executors = REDIS_CONN.smembers("TASKEXE")
         now = datetime.now().timestamp()
         for task_executor_id in task_executors:
-            heartbeats = REDIS_CONN.zrangebyscore(task_executor_id, now - 60 * 30, now)
+            heartbeats = REDIS_CONN.zrangebyscore(task_executor_id, now - 60*30, now)
             heartbeats = [json.loads(heartbeat) for heartbeat in heartbeats]
             task_executor_heartbeats[task_executor_id] = heartbeats
     except Exception:
@@ -178,11 +178,6 @@ def healthz():
     return jsonify(result), (200 if all_ok else 500)
 
 
-@manager.route("/ping", methods=["GET"]) # noqa: F821
-def ping():
-    return "pong", 200
-
-
 @manager.route("/new_token", methods=["POST"])  # noqa: F821
 @login_required
 def new_token():
@@ -274,8 +269,7 @@ def token_list():
         objs = [o.to_dict() for o in objs]
         for o in objs:
             if not o["beta"]:
-                o["beta"] = generate_confirmation_token(generate_confirmation_token(tenants[0].tenant_id)).replace(
-                    "ragflow-", "")[:32]
+                o["beta"] = generate_confirmation_token(generate_confirmation_token(tenants[0].tenant_id)).replace("ragflow-", "")[:32]
                 APITokenService.filter_update([APIToken.tenant_id == tenant_id, APIToken.token == o["token"]], o)
         return get_json_result(data=objs)
     except Exception as e:

api/apps/tenant_app.py

@@ -70,8 +70,7 @@ def create(tenant_id):
             return get_data_error_result(message=f"{invite_user_email} is already in the team.")
         if user_tenant_role == UserTenantRole.OWNER:
             return get_data_error_result(message=f"{invite_user_email} is the owner of the team.")
-        return get_data_error_result(
-            message=f"{invite_user_email} is in the team, but the role: {user_tenant_role} is invalid.")
+        return get_data_error_result(message=f"{invite_user_email} is in the team, but the role: {user_tenant_role} is invalid.")
 
     UserTenantService.save(
         id=get_uuid(),
@@ -133,8 +132,7 @@ def tenant_list():
 @login_required
 def agree(tenant_id):
     try:
-        UserTenantService.filter_update([UserTenant.tenant_id == tenant_id, UserTenant.user_id == current_user.id],
-                                        {"role": UserTenantRole.NORMAL})
+        UserTenantService.filter_update([UserTenant.tenant_id == tenant_id, UserTenant.user_id == current_user.id], {"role": UserTenantRole.NORMAL})
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)

api/common/check_team_permission.py

@@ -1,59 +0,0 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-
-from api.db import TenantPermission
-from api.db.db_models import File, Knowledgebase
-from api.db.services.file_service import FileService
-from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.user_service import TenantService
-
-
-def check_kb_team_permission(kb: dict | Knowledgebase, other: str) -> bool:
-    kb = kb.to_dict() if isinstance(kb, Knowledgebase) else kb
-
-    kb_tenant_id = kb["tenant_id"]
-
-    if kb_tenant_id == other:
-        return True
-
-    if kb["permission"] != TenantPermission.TEAM:
-        return False
-
-    joined_tenants = TenantService.get_joined_tenants_by_user_id(other)
-    return any(tenant["tenant_id"] == kb_tenant_id for tenant in joined_tenants)
-
-
-def check_file_team_permission(file: dict | File, other: str) -> bool:
-    file = file.to_dict() if isinstance(file, File) else file
-
-    file_tenant_id = file["tenant_id"]
-    if file_tenant_id == other:
-        return True
-
-    file_id = file["id"]
-
-    kb_ids = [kb_info["kb_id"] for kb_info in FileService.get_kb_id_by_file_id(file_id)]
-
-    for kb_id in kb_ids:
-        ok, kb = KnowledgebaseService.get_by_id(kb_id)
-        if not ok:
-            continue
-
-        if check_kb_team_permission(kb, other):
-            return True
-
-    return False

api/db/services/dialog_service.py

@@ -370,7 +370,7 @@ def chat(dialog, messages, stream=True, **kwargs):
         chat_mdl.bind_tools(toolcall_session, tools)
     bind_models_ts = timer()
 
-    retriever = settings.retriever
+    retriever = settings.retrievaler
     questions = [m["content"] for m in messages if m["role"] == "user"][-3:]
     attachments = kwargs["doc_ids"].split(",") if "doc_ids" in kwargs else []
     if "doc_ids" in messages[-1]:
@@ -472,7 +472,7 @@ def chat(dialog, messages, stream=True, **kwargs):
                 kbinfos["chunks"].extend(tav_res["chunks"])
                 kbinfos["doc_aggs"].extend(tav_res["doc_aggs"])
             if prompt_config.get("use_kg"):
-                ck = settings.kg_retriever.retrieval(" ".join(questions), tenant_ids, dialog.kb_ids, embd_mdl,
+                ck = settings.kg_retrievaler.retrieval(" ".join(questions), tenant_ids, dialog.kb_ids, embd_mdl,
                                                        LLMBundle(dialog.tenant_id, LLMType.CHAT))
                 if ck["content_with_weight"]:
                     kbinfos["chunks"].insert(0, ck)
@@ -658,7 +658,7 @@ Please write the SQL, only SQL, without any other explanations or text.
 
         logging.debug(f"{question} get SQL(refined): {sql}")
         tried_times += 1
-        return settings.retriever.sql_retrieval(sql, format="json"), sql
+        return settings.retrievaler.sql_retrieval(sql, format="json"), sql
 
     tbl, sql = get_table()
     if tbl is None:
@@ -752,7 +752,7 @@ def ask(question, kb_ids, tenant_id, chat_llm_name=None, search_config={}):
     embedding_list = list(set([kb.embd_id for kb in kbs]))
 
     is_knowledge_graph = all([kb.parser_id == ParserType.KG for kb in kbs])
-    retriever = settings.retriever if not is_knowledge_graph else settings.kg_retriever
+    retriever = settings.retrievaler if not is_knowledge_graph else settings.kg_retrievaler
 
     embd_mdl = LLMBundle(tenant_id, LLMType.EMBEDDING, embedding_list[0])
     chat_mdl = LLMBundle(tenant_id, LLMType.CHAT, chat_llm_name)
@@ -848,7 +848,7 @@ def gen_mindmap(question, kb_ids, tenant_id, search_config={}):
             if not doc_ids:
                 doc_ids = None
 
-    ranks = settings.retriever.retrieval(
+    ranks = settings.retrievaler.retrieval(
         question=question,
         embd_mdl=embd_mdl,
         tenant_ids=tenant_ids,

api/db/services/knowledgebase_service.py

@@ -379,7 +379,6 @@ class KnowledgebaseService(CommonService):
         #     name: Optional name filter
         # Returns:
         #     List of knowledge bases
-        #     Total count of knowledge bases
         kbs = cls.model.select()
         if id:
             kbs = kbs.where(cls.model.id == id)
@@ -391,7 +390,6 @@ class KnowledgebaseService(CommonService):
                 cls.model.tenant_id == user_id))
             & (cls.model.status == StatusEnum.VALID.value)
         )
-
         if desc:
             kbs = kbs.order_by(cls.model.getter_by(orderby).desc())
         else:
@@ -399,7 +397,7 @@ class KnowledgebaseService(CommonService):
 
         kbs = kbs.paginate(page_number, items_per_page)
 
-        return list(kbs.dicts()), kbs.count()
+        return list(kbs.dicts())
 
     @classmethod
     @DB.connection_context()

api/db/services/mcp_server_service.py

@@ -33,8 +33,7 @@ class MCPServerService(CommonService):
 
     @classmethod
     @DB.connection_context()
-    def get_servers(cls, tenant_id: str, id_list: list[str] | None, page_number, items_per_page, orderby, desc,
-                    keywords):
+    def get_servers(cls, tenant_id: str, id_list: list[str] | None, page_number, items_per_page, orderby, desc, keywords):
         """Retrieve all MCP servers associated with a tenant.
 
         This method fetches all MCP servers for a given tenant, ordered by creation time.

api/db/services/search_service.py

@@ -94,8 +94,7 @@ class SearchService(CommonService):
         query = (
             cls.model.select(*fields)
             .join(User, on=(cls.model.tenant_id == User.id))
-            .where(((cls.model.tenant_id.in_(joined_tenant_ids)) | (cls.model.tenant_id == user_id)) & (
-                        cls.model.status == StatusEnum.VALID.value))
+            .where(((cls.model.tenant_id.in_(joined_tenant_ids)) | (cls.model.tenant_id == user_id)) & (cls.model.status == StatusEnum.VALID.value))
         )
 
         if keywords:

api/db/services/task_service.py

@@ -165,7 +165,7 @@ class TaskService(CommonService):
         ]
         tasks = (
             cls.model.select(*fields).order_by(cls.model.from_page.asc(), cls.model.create_time.desc())
-            .where(cls.model.doc_id == doc_id)
+                .where(cls.model.doc_id == doc_id)
         )
         tasks = list(tasks.dicts())
         if not tasks:
@@ -205,18 +205,18 @@ class TaskService(CommonService):
                 cls.model.select(
                     *[Document.id, Document.kb_id, Document.location, File.parent_id]
                 )
-                .join(Document, on=(cls.model.doc_id == Document.id))
-                .join(
+                    .join(Document, on=(cls.model.doc_id == Document.id))
+                    .join(
                     File2Document,
                     on=(File2Document.document_id == Document.id),
                     join_type=JOIN.LEFT_OUTER,
                 )
-                .join(
+                    .join(
                     File,
                     on=(File2Document.file_id == File.id),
                     join_type=JOIN.LEFT_OUTER,
                 )
-                .where(
+                    .where(
                     Document.status == StatusEnum.VALID.value,
                     Document.run == TaskStatus.RUNNING.value,
                     ~(Document.type == FileType.VIRTUAL.value),
@@ -294,8 +294,8 @@ class TaskService(CommonService):
                 cls.model.update(progress=prog).where(
                     (cls.model.id == id) &
                     (
-                            (cls.model.progress != -1) &
-                            ((prog == -1) | (prog > cls.model.progress))
+                        (cls.model.progress != -1) &
+                        ((prog == -1) | (prog > cls.model.progress))
                     )
                 ).execute()
         else:
@@ -343,7 +343,6 @@ def queue_tasks(doc: dict, bucket: str, name: str, priority: int):
         - Task digests are calculated for optimization and reuse
         - Previous task chunks may be reused if available
     """
-
     def new_task():
         return {
             "id": get_uuid(),
@@ -516,7 +515,7 @@ def queue_dataflow(tenant_id:str, flow_id:str, task_id:str, doc_id:str=CANVAS_DE
     task["file"] = file
 
     if not REDIS_CONN.queue_product(
-            get_svr_queue_name(priority), message=task
+        get_svr_queue_name(priority), message=task
     ):
         return False, "Can't access Redis. Please check the Redis' status."
 

api/db/services/tenant_llm_service.py

@@ -57,10 +57,8 @@ class TenantLLMService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_my_llms(cls, tenant_id):
-        fields = [cls.model.llm_factory, LLMFactories.logo, LLMFactories.tags, cls.model.model_type, cls.model.llm_name,
-                  cls.model.used_tokens]
-        objs = cls.model.select(*fields).join(LLMFactories, on=(cls.model.llm_factory == LLMFactories.name)).where(
-            cls.model.tenant_id == tenant_id, ~cls.model.api_key.is_null()).dicts()
+        fields = [cls.model.llm_factory, LLMFactories.logo, LLMFactories.tags, cls.model.model_type, cls.model.llm_name, cls.model.used_tokens]
+        objs = cls.model.select(*fields).join(LLMFactories, on=(cls.model.llm_factory == LLMFactories.name)).where(cls.model.tenant_id == tenant_id, ~cls.model.api_key.is_null()).dicts()
 
         return list(objs)
 
@@ -124,8 +122,7 @@ class TenantLLMService(CommonService):
                     model_config = {"llm_factory": llm[0].fid, "api_key": "", "llm_name": mdlnm, "api_base": ""}
             if not model_config:
                 if mdlnm == "flag-embedding":
-                    model_config = {"llm_factory": "Tongyi-Qianwen", "api_key": "", "llm_name": llm_name,
-                                    "api_base": ""}
+                    model_config = {"llm_factory": "Tongyi-Qianwen", "api_key": "", "llm_name": llm_name, "api_base": ""}
                 else:
                     if not mdlnm:
                         raise LookupError(f"Type of {llm_type} model is not set.")
@@ -140,33 +137,27 @@ class TenantLLMService(CommonService):
         if llm_type == LLMType.EMBEDDING.value:
             if model_config["llm_factory"] not in EmbeddingModel:
                 return
-            return EmbeddingModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"],
-                                                               base_url=model_config["api_base"])
+            return EmbeddingModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
 
         if llm_type == LLMType.RERANK:
             if model_config["llm_factory"] not in RerankModel:
                 return
-            return RerankModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"],
-                                                            base_url=model_config["api_base"])
+            return RerankModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
 
         if llm_type == LLMType.IMAGE2TEXT.value:
             if model_config["llm_factory"] not in CvModel:
                 return
-            return CvModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], lang,
-                                                        base_url=model_config["api_base"], **kwargs)
+            return CvModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], lang, base_url=model_config["api_base"], **kwargs)
 
         if llm_type == LLMType.CHAT.value:
             if model_config["llm_factory"] not in ChatModel:
                 return
-            return ChatModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"],
-                                                          base_url=model_config["api_base"], **kwargs)
+            return ChatModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"], **kwargs)
 
         if llm_type == LLMType.SPEECH2TEXT:
             if model_config["llm_factory"] not in Seq2txtModel:
                 return
-            return Seq2txtModel[model_config["llm_factory"]](key=model_config["api_key"],
-                                                             model_name=model_config["llm_name"], lang=lang,
-                                                             base_url=model_config["api_base"])
+            return Seq2txtModel[model_config["llm_factory"]](key=model_config["api_key"], model_name=model_config["llm_name"], lang=lang, base_url=model_config["api_base"])
         if llm_type == LLMType.TTS:
             if model_config["llm_factory"] not in TTSModel:
                 return
@@ -203,14 +194,11 @@ class TenantLLMService(CommonService):
         try:
             num = (
                 cls.model.update(used_tokens=cls.model.used_tokens + used_tokens)
-                .where(cls.model.tenant_id == tenant_id, cls.model.llm_name == llm_name,
-                       cls.model.llm_factory == llm_factory if llm_factory else True)
+                .where(cls.model.tenant_id == tenant_id, cls.model.llm_name == llm_name, cls.model.llm_factory == llm_factory if llm_factory else True)
                 .execute()
             )
         except Exception:
-            logging.exception(
-                "TenantLLMService.increase_usage got exception,Failed to update used_tokens for tenant_id=%s, llm_name=%s",
-                tenant_id, llm_name)
+            logging.exception("TenantLLMService.increase_usage got exception,Failed to update used_tokens for tenant_id=%s, llm_name=%s", tenant_id, llm_name)
             return 0
 
         return num
@@ -218,9 +206,7 @@ class TenantLLMService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_openai_models(cls):
-        objs = cls.model.select().where((cls.model.llm_factory == "OpenAI"),
-                                        ~(cls.model.llm_name == "text-embedding-3-small"),
-                                        ~(cls.model.llm_name == "text-embedding-3-large")).dicts()
+        objs = cls.model.select().where((cls.model.llm_factory == "OpenAI"), ~(cls.model.llm_name == "text-embedding-3-small"), ~(cls.model.llm_name == "text-embedding-3-large")).dicts()
         return list(objs)
 
     @classmethod
@@ -264,9 +250,8 @@ class LLM4Tenant:
         langfuse_keys = TenantLangfuseService.filter_by_tenant(tenant_id=tenant_id)
         self.langfuse = None
         if langfuse_keys:
-            langfuse = Langfuse(public_key=langfuse_keys.public_key, secret_key=langfuse_keys.secret_key,
-                                host=langfuse_keys.host)
+            langfuse = Langfuse(public_key=langfuse_keys.public_key, secret_key=langfuse_keys.secret_key, host=langfuse_keys.host)
             if langfuse.auth_check():
                 self.langfuse = langfuse
                 trace_id = self.langfuse.create_trace_id()
-                self.trace_context = {"trace_id": trace_id}
+                self.trace_context = {"trace_id": trace_id}

api/db/services/user_canvas_version.py

@@ -2,22 +2,22 @@ from api.db.db_models import UserCanvasVersion, DB
 from api.db.services.common_service import CommonService
 from peewee import DoesNotExist
 
-
 class UserCanvasVersionService(CommonService):
     model = UserCanvasVersion
-
+    
+    
     @classmethod
     @DB.connection_context()
     def list_by_canvas_id(cls, user_canvas_id):
         try:
             user_canvas_version = cls.model.select(
-                *[cls.model.id,
-                  cls.model.create_time,
-                  cls.model.title,
-                  cls.model.create_date,
-                  cls.model.update_date,
-                  cls.model.user_canvas_id,
-                  cls.model.update_time]
+                *[cls.model.id, 
+                cls.model.create_time,
+                cls.model.title,
+                cls.model.create_date, 
+                cls.model.update_date,
+                cls.model.user_canvas_id, 
+                cls.model.update_time]
             ).where(cls.model.user_canvas_id == user_canvas_id)
             return user_canvas_version
         except DoesNotExist:
@@ -46,16 +46,18 @@ class UserCanvasVersionService(CommonService):
     @DB.connection_context()
     def delete_all_versions(cls, user_canvas_id):
         try:
-            user_canvas_version = cls.model.select().where(cls.model.user_canvas_id == user_canvas_id).order_by(
-                cls.model.create_time.desc())
+            user_canvas_version = cls.model.select().where(cls.model.user_canvas_id == user_canvas_id).order_by(cls.model.create_time.desc())
             if user_canvas_version.count() > 20:
                 delete_ids = []
                 for i in range(20, user_canvas_version.count()):
                     delete_ids.append(user_canvas_version[i].id)
-
+                
                 cls.delete_by_ids(delete_ids)
             return True
         except DoesNotExist:
             return None
         except Exception:
             return None
+
+
+

api/db/services/user_service.py

@@ -315,4 +315,4 @@ class UserTenantService(CommonService):
             ).first()
             return user_tenant
         except peewee.DoesNotExist:
-            return None
+            return None

api/settings.py

@@ -65,8 +65,8 @@ OAUTH_CONFIG = None
 DOC_ENGINE = None
 docStoreConn = None
 
-retriever = None
-kg_retriever = None
+retrievaler = None
+kg_retrievaler = None
 
 # user registration switch
 REGISTER_ENABLED = 1
@@ -174,7 +174,7 @@ def init_settings():
 
     OAUTH_CONFIG = get_base_config("oauth", {})
 
-    global DOC_ENGINE, docStoreConn, retriever, kg_retriever
+    global DOC_ENGINE, docStoreConn, retrievaler, kg_retrievaler
     DOC_ENGINE = os.environ.get("DOC_ENGINE", "elasticsearch")
     # DOC_ENGINE = os.environ.get('DOC_ENGINE', "opensearch")
     lower_case_doc_engine = DOC_ENGINE.lower()
@@ -187,10 +187,10 @@ def init_settings():
     else:
         raise Exception(f"Not supported doc engine: {DOC_ENGINE}")
 
-    retriever = search.Dealer(docStoreConn)
+    retrievaler = search.Dealer(docStoreConn)
     from graphrag import search as kg_search
 
-    kg_retriever = kg_search.KGSearch(docStoreConn)
+    kg_retrievaler = kg_search.KGSearch(docStoreConn)
 
     if int(os.environ.get("SANDBOX_ENABLED", "0")):
         global SANDBOX_HOST

api/utils/api_utils.py

@@ -60,7 +60,6 @@ from rag.utils.mcp_tool_call_conn import MCPToolCallSession, close_multiple_mcp_
 
 requests.models.complexjson.dumps = functools.partial(json.dumps, cls=CustomJSONEncoder)
 
-
 def serialize_for_json(obj):
     """
     Recursively serialize objects to make them JSON serializable.
@@ -69,8 +68,8 @@ def serialize_for_json(obj):
     if hasattr(obj, '__dict__'):
         # For objects with __dict__, try to serialize their attributes
         try:
-            return {key: serialize_for_json(value) for key, value in obj.__dict__.items()
-                    if not key.startswith('_')}
+            return {key: serialize_for_json(value) for key, value in obj.__dict__.items() 
+                   if not key.startswith('_')}
         except (AttributeError, TypeError):
             return str(obj)
     elif hasattr(obj, '__name__'):
@@ -86,7 +85,6 @@ def serialize_for_json(obj):
         # Fallback: convert to string representation
         return str(obj)
 
-
 def request(**kwargs):
     sess = requests.Session()
     stream = kwargs.pop("stream", sess.stream)
@@ -107,8 +105,7 @@ def request(**kwargs):
                         settings.HTTP_APP_KEY.encode("ascii"),
                         prepped.path_url.encode("ascii"),
                         prepped.body if kwargs.get("json") else b"",
-                        urlencode(sorted(kwargs["data"].items()), quote_via=quote, safe="-._~").encode(
-                            "ascii") if kwargs.get("data") and isinstance(kwargs["data"], dict) else b"",
+                        urlencode(sorted(kwargs["data"].items()), quote_via=quote, safe="-._~").encode("ascii") if kwargs.get("data") and isinstance(kwargs["data"], dict) else b"",
                     ]
                 ),
                 "sha1",
@@ -130,7 +127,7 @@ def request(**kwargs):
 def get_exponential_backoff_interval(retries, full_jitter=False):
     """Calculate the exponential backoff wait time."""
     # Will be zero if factor equals 0
-    countdown = min(REQUEST_MAX_WAIT_SEC, REQUEST_WAIT_SEC * (2 ** retries))
+    countdown = min(REQUEST_MAX_WAIT_SEC, REQUEST_WAIT_SEC * (2**retries))
     # Full jitter according to
     # https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
     if full_jitter:
@@ -161,12 +158,11 @@ def server_error_response(e):
     if len(e.args) > 1:
         try:
             serialized_data = serialize_for_json(e.args[1])
-            return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message=repr(e.args[0]), data=serialized_data)
+            return get_json_result(code= settings.RetCode.EXCEPTION_ERROR, message=repr(e.args[0]), data=serialized_data)
         except Exception:
             return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message=repr(e.args[0]), data=None)
     if repr(e).find("index_not_found_exception") >= 0:
-        return get_json_result(code=settings.RetCode.EXCEPTION_ERROR,
-                               message="No chunk found, please upload file and parse it.")
+        return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message="No chunk found, please upload file and parse it.")
 
     return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message=repr(e))
 
@@ -211,8 +207,7 @@ def validate_request(*args, **kwargs):
                 if no_arguments:
                     error_string += "required argument are missing: {}; ".format(",".join(no_arguments))
                 if error_arguments:
-                    error_string += "required argument values: {}".format(
-                        ",".join(["{}={}".format(a[0], a[1]) for a in error_arguments]))
+                    error_string += "required argument values: {}".format(",".join(["{}={}".format(a[0], a[1]) for a in error_arguments]))
                 return get_json_result(code=settings.RetCode.ARGUMENT_ERROR, message=error_string)
             return func(*_args, **_kwargs)
 
@@ -227,8 +222,7 @@ def not_allowed_parameters(*params):
             input_arguments = flask_request.json or flask_request.form.to_dict()
             for param in params:
                 if param in input_arguments:
-                    return get_json_result(code=settings.RetCode.ARGUMENT_ERROR,
-                                           message=f"Parameter {param} isn't allowed")
+                    return get_json_result(code=settings.RetCode.ARGUMENT_ERROR, message=f"Parameter {param} isn't allowed")
             return f(*args, **kwargs)
 
         return wrapper
@@ -245,7 +239,6 @@ def active_required(f):
         if not usr or not usr.is_active == ActiveEnum.ACTIVE.value:
             return get_json_result(code=settings.RetCode.FORBIDDEN, message="User isn't active, please activate first.")
         return f(*args, **kwargs)
-
     return wrapper
 
 
@@ -266,7 +259,7 @@ def send_file_in_mem(data, filename):
     return send_file(f, as_attachment=True, attachment_filename=filename)
 
 
-def get_json_result(code: settings.RetCode = settings.RetCode.SUCCESS, message="success", data=None):
+def get_json_result(code=settings.RetCode.SUCCESS, message="success", data=None):
     response = {"code": code, "message": message, "data": data}
     return jsonify(response)
 
@@ -321,7 +314,7 @@ def construct_result(code=settings.RetCode.DATA_ERROR, message="data is missing"
     return jsonify(response)
 
 
-def construct_json_result(code: settings.RetCode = settings.RetCode.SUCCESS, message="success", data=None):
+def construct_json_result(code=settings.RetCode.SUCCESS, message="success", data=None):
     if data is None:
         return jsonify({"code": code, "message": message})
     else:
@@ -354,39 +347,27 @@ def token_required(func):
         token = authorization_list[1]
         objs = APIToken.query(token=token)
         if not objs:
-            return get_json_result(data=False, message="Authentication error: API key is invalid!",
-                                   code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_json_result(data=False, message="Authentication error: API key is invalid!", code=settings.RetCode.AUTHENTICATION_ERROR)
         kwargs["tenant_id"] = objs[0].tenant_id
         return func(*args, **kwargs)
 
     return decorated_function
 
 
-def get_result(code=settings.RetCode.SUCCESS, message="", data=None, total=None):
-    """
-    Standard API response format:
-    {
-        "code": 0,
-        "data": [...],        # List or object, backward compatible
-        "total": 47,          # Optional field for pagination
-        "message": "..."      # Error or status message
-    }
-    """
-    response = {"code": code}
-
-    if code == settings.RetCode.SUCCESS:
+def get_result(code=settings.RetCode.SUCCESS, message="", data=None):
+    if code == 0:
         if data is not None:
-            response["data"] = data
-        if total is not None:
-            response["total_datasets"] = total
+            response = {"code": code, "data": data}
+        else:
+            response = {"code": code}
     else:
-        response["message"] = message or "Error"
-
+        response = {"code": code, "message": message}
     return jsonify(response)
 
+
 def get_error_data_result(
-        message="Sorry! Data missing!",
-        code=settings.RetCode.DATA_ERROR,
+    message="Sorry! Data missing!",
+    code=settings.RetCode.DATA_ERROR,
 ):
     result_dict = {"code": code, "message": message}
     response = {}
@@ -421,8 +402,7 @@ def get_parser_config(chunk_method, parser_config):
 
     # Define default configurations for each chunking method
     key_mapping = {
-        "naive": {"chunk_token_num": 512, "delimiter": r"\n", "html4excel": False, "layout_recognize": "DeepDOC",
-                  "raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
+        "naive": {"chunk_token_num": 512, "delimiter": r"\n", "html4excel": False, "layout_recognize": "DeepDOC", "raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
         "qa": {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
         "tag": None,
         "resume": None,
@@ -461,16 +441,16 @@ def get_parser_config(chunk_method, parser_config):
 
 
 def get_data_openai(
-        id=None,
-        created=None,
-        model=None,
-        prompt_tokens=0,
-        completion_tokens=0,
-        content=None,
-        finish_reason=None,
-        object="chat.completion",
-        param=None,
-        stream=False
+    id=None,
+    created=None,
+    model=None,
+    prompt_tokens=0,
+    completion_tokens=0,
+    content=None,
+    finish_reason=None,
+    object="chat.completion",
+    param=None,
+    stream=False
 ):
     total_tokens = prompt_tokens + completion_tokens
 
@@ -582,9 +562,7 @@ def verify_embedding_availability(embd_id: str, tenant_id: str) -> tuple[bool, R
         in_llm_service = bool(LLMService.query(llm_name=llm_name, fid=llm_factory, model_type="embedding"))
 
         tenant_llms = TenantLLMService.get_my_llms(tenant_id=tenant_id)
-        is_tenant_model = any(
-            llm["llm_name"] == llm_name and llm["llm_factory"] == llm_factory and llm["model_type"] == "embedding" for
-            llm in tenant_llms)
+        is_tenant_model = any(llm["llm_name"] == llm_name and llm["llm_factory"] == llm_factory and llm["model_type"] == "embedding" for llm in tenant_llms)
 
         is_builtin_model = embd_id in settings.BUILTIN_EMBEDDING_MODELS
         if not (is_builtin_model or is_tenant_model or in_llm_service):
@@ -815,9 +793,7 @@ async def is_strong_enough(chat_model, embedding_model):
                 _ = await trio.to_thread.run_sync(lambda: embedding_model.encode(["Are you strong enough!?"]))
         if chat_model:
             with trio.fail_after(30):
-                res = await trio.to_thread.run_sync(lambda: chat_model.chat("Nothing special.", [{"role": "user",
-                                                                                                  "content": "Are you strong enough!?"}],
-                                                                            {}))
+                res = await trio.to_thread.run_sync(lambda: chat_model.chat("Nothing special.", [{"role": "user", "content": "Are you strong enough!?"}], {}))
             if res.find("**ERROR**") >= 0:
                 raise Exception(res)
 

api/utils/common.py

@@ -21,26 +21,3 @@ def string_to_bytes(string):
 
 def bytes_to_string(byte):
     return byte.decode(encoding="utf-8")
-
-
-def convert_bytes(size_in_bytes: int) -> str:
-    """
-    Format size in bytes.
-    """
-    if size_in_bytes == 0:
-        return "0 B"
-
-    units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
-    i = 0
-    size = float(size_in_bytes)
-
-    while size >= 1024 and i < len(units) - 1:
-        size /= 1024
-        i += 1
-
-    if i == 0 or size >= 100:
-        return f"{size:.0f} {units[i]}"
-    elif size >= 10:
-        return f"{size:.1f} {units[i]}"
-    else:
-        return f"{size:.2f} {units[i]}"

api/utils/health_utils.py

@@ -13,17 +13,14 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import os
-import requests
+
+
 from timeit import default_timer as timer
 
 from api import settings
 from api.db.db_models import DB
-from rag import settings as rag_settings
 from rag.utils.redis_conn import REDIS_CONN
 from rag.utils.storage_factory import STORAGE_IMPL
-from rag.utils.es_conn import ESConnection
-from rag.utils.infinity_conn import InfinityConnection
 
 
 def _ok_nok(ok: bool) -> str:
@@ -68,96 +65,6 @@ def check_storage() -> tuple[bool, dict]:
         return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)}
 
 
-def get_es_cluster_stats() -> dict:
-    doc_engine = os.getenv('DOC_ENGINE', 'elasticsearch')
-    if doc_engine != 'elasticsearch':
-        raise Exception("Elasticsearch is not in use.")
-    try:
-        return {
-            "alive": True,
-            "message": ESConnection().get_cluster_stats()
-        }
-    except Exception as e:
-        return {
-            "alive": False,
-            "message": f"error: {str(e)}",
-        }
-
-
-def get_infinity_status():
-    doc_engine = os.getenv('DOC_ENGINE', 'elasticsearch')
-    if doc_engine != 'infinity':
-        raise Exception("Infinity is not in use.")
-    try:
-        return {
-            "alive": True,
-            "message": InfinityConnection().health()
-        }
-    except Exception as e:
-        return {
-            "alive": False,
-            "message": f"error: {str(e)}",
-        }
-
-
-def get_mysql_status():
-    try:
-        cursor = DB.execute_sql("SHOW PROCESSLIST;")
-        res_rows = cursor.fetchall()
-        headers = ['id', 'user', 'host', 'db', 'command', 'time', 'state', 'info']
-        cursor.close()
-        return {
-            "alive": True,
-            "message": [dict(zip(headers, r)) for r in res_rows]
-        }
-    except Exception as e:
-        return {
-            "alive": False,
-            "message": f"error: {str(e)}",
-        }
-
-
-def check_minio_alive():
-    start_time = timer()
-    try:
-        response = requests.get(f'http://{rag_settings.MINIO["host"]}/minio/health/live')
-        if response.status_code == 200:
-            return {'alive': True, "message": f"Confirm elapsed: {(timer() - start_time) * 1000.0:.1f} ms."}
-        else:
-            return {'alive': False, "message": f"Confirm elapsed: {(timer() - start_time) * 1000.0:.1f} ms."}
-    except Exception as e:
-        return {
-            "alive": False,
-            "message": f"error: {str(e)}",
-        }
-
-
-def get_redis_info():
-    try:
-        return {
-            "alive": True,
-            "message": REDIS_CONN.info()
-        }
-    except Exception as e:
-        return {
-            "alive": False,
-            "message": f"error: {str(e)}",
-        }
-
-
-def check_ragflow_server_alive():
-    start_time = timer()
-    try:
-        response = requests.get(f'http://{settings.HOST_IP}:{settings.HOST_PORT}/v1/system/ping')
-        if response.status_code == 200:
-            return {'alive': True, "message": f"Confirm elapsed: {(timer() - start_time) * 1000.0:.1f} ms."}
-        else:
-            return {'alive': False, "message": f"Confirm elapsed: {(timer() - start_time) * 1000.0:.1f} ms."}
-    except Exception as e:
-        return {
-            "alive": False,
-            "message": f"error: {str(e)}",
-        }
 
 
 def run_health_checks() -> tuple[dict, bool]:

conf/llm_factories.json

@@ -3533,13 +3533,6 @@
                     "model_type": "chat",
                     "is_tools": true
                 },
-                {
-                    "llm_name": "claude-sonnet-4-5-20250929",
-                    "tags": "LLM,CHAT,IMAGE2TEXT,200k",
-                    "max_tokens": 204800,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
                 {
                     "llm_name": "claude-sonnet-4-20250514",
                     "tags": "LLM,CHAT,IMAGE2TEXT,200k",
@@ -4869,280 +4862,6 @@
                     "max_tokens": 8000,
                     "model_type": "chat",
                     "is_tools": true
-                },
-                {
-                    "llm_name": "LongCat-Flash-Thinking",
-                    "tags": "LLM,CHAT,8000",
-                    "max_tokens": 8000,
-                    "model_type": "chat",
-                    "is_tools": true
-                }
-            ]
-        },
-        {
-            "name": "DeerAPI",
-            "logo": "",
-            "tags": "LLM,TEXT EMBEDDING,IMAGE2TEXT",
-            "status": "1",
-            "llm": [
-                {
-                    "llm_name": "gpt-5-chat-latest",
-                    "tags": "LLM,CHAT,400k",
-                    "max_tokens": 400000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "chatgpt-4o-latest",
-                    "tags": "LLM,CHAT,128k",
-                    "max_tokens": 128000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gpt-5-mini",
-                    "tags": "LLM,CHAT,400k",
-                    "max_tokens": 400000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gpt-5-nano",
-                    "tags": "LLM,CHAT,400k",
-                    "max_tokens": 400000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gpt-5",
-                    "tags": "LLM,CHAT,400k",
-                    "max_tokens": 400000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gpt-4.1-mini",
-                    "tags": "LLM,CHAT,1M",
-                    "max_tokens": 1047576,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gpt-4.1-nano",
-                    "tags": "LLM,CHAT,1M",
-                    "max_tokens": 1047576,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gpt-4.1",
-                    "tags": "LLM,CHAT,1M",
-                    "max_tokens": 1047576,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gpt-4o-mini",
-                    "tags": "LLM,CHAT,128k",
-                    "max_tokens": 128000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "o4-mini-2025-04-16",
-                    "tags": "LLM,CHAT,200k",
-                    "max_tokens": 200000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "o3-pro-2025-06-10",
-                    "tags": "LLM,CHAT,200k",
-                    "max_tokens": 200000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "claude-opus-4-1-20250805",
-                    "tags": "LLM,CHAT,200k,IMAGE2TEXT",
-                    "max_tokens": 200000,
-                    "model_type": "image2text",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "claude-opus-4-1-20250805-thinking",
-                    "tags": "LLM,CHAT,200k,IMAGE2TEXT",
-                    "max_tokens": 200000,
-                    "model_type": "image2text",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "claude-sonnet-4-20250514",
-                    "tags": "LLM,CHAT,200k,IMAGE2TEXT",
-                    "max_tokens": 200000,
-                    "model_type": "image2text",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "claude-sonnet-4-20250514-thinking",
-                    "tags": "LLM,CHAT,200k,IMAGE2TEXT",
-                    "max_tokens": 200000,
-                    "model_type": "image2text",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "claude-3-7-sonnet-latest",
-                    "tags": "LLM,CHAT,200k",
-                    "max_tokens": 200000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "claude-3-5-haiku-latest",
-                    "tags": "LLM,CHAT,200k",
-                    "max_tokens": 200000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gemini-2.5-pro",
-                    "tags": "LLM,CHAT,1M,IMAGE2TEXT",
-                    "max_tokens": 1000000,
-                    "model_type": "image2text",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gemini-2.5-flash",
-                    "tags": "LLM,CHAT,1M,IMAGE2TEXT",
-                    "max_tokens": 1000000,
-                    "model_type": "image2text",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gemini-2.5-flash-lite",
-                    "tags": "LLM,CHAT,1M,IMAGE2TEXT",
-                    "max_tokens": 1000000,
-                    "model_type": "image2text",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "gemini-2.0-flash",
-                    "tags": "LLM,CHAT,1M,IMAGE2TEXT",
-                    "max_tokens": 1000000,
-                    "model_type": "image2text",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "grok-4-0709",
-                    "tags": "LLM,CHAT,131k",
-                    "max_tokens": 131072,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "grok-3",
-                    "tags": "LLM,CHAT,131k",
-                    "max_tokens": 131072,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "grok-3-mini",
-                    "tags": "LLM,CHAT,131k",
-                    "max_tokens": 131072,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "grok-2-image-1212",
-                    "tags": "LLM,CHAT,32k,IMAGE2TEXT",
-                    "max_tokens": 32768,
-                    "model_type": "image2text",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "deepseek-v3.1",
-                    "tags": "LLM,CHAT,64k",
-                    "max_tokens": 64000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "deepseek-v3",
-                    "tags": "LLM,CHAT,64k",
-                    "max_tokens": 64000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "deepseek-r1-0528",
-                    "tags": "LLM,CHAT,164k",
-                    "max_tokens": 164000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "deepseek-chat",
-                    "tags": "LLM,CHAT,32k",
-                    "max_tokens": 32000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "deepseek-reasoner",
-                    "tags": "LLM,CHAT,64k",
-                    "max_tokens": 64000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "qwen3-30b-a3b",
-                    "tags": "LLM,CHAT,128k",
-                    "max_tokens": 128000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "qwen3-coder-plus-2025-07-22",
-                    "tags": "LLM,CHAT,128k",
-                    "max_tokens": 128000,
-                    "model_type": "chat",
-                    "is_tools": true
-                },
-                {
-                    "llm_name": "text-embedding-ada-002",
-                    "tags": "TEXT EMBEDDING,8K",
-                    "max_tokens": 8191,
-                    "model_type": "embedding",
-                    "is_tools": false
-                },
-                {
-                    "llm_name": "text-embedding-3-small",
-                    "tags": "TEXT EMBEDDING,8K",
-                    "max_tokens": 8191,
-                    "model_type": "embedding",
-                    "is_tools": false
-                },
-                {
-                    "llm_name": "text-embedding-3-large",
-                    "tags": "TEXT EMBEDDING,8K",
-                    "max_tokens": 8191,
-                    "model_type": "embedding",
-                    "is_tools": false
-                },
-                {
-                    "llm_name": "whisper-1",
-                    "tags": "SPEECH2TEXT",
-                    "max_tokens": 26214400,
-                    "model_type": "speech2text",
-                    "is_tools": false
-                },
-                {
-                    "llm_name": "tts-1",
-                    "tags": "TTS",
-                    "max_tokens": 2048,
-                    "model_type": "tts",
-                    "is_tools": false
                 }
             ]
         }

deepdoc/parser/pdf_parser.py

@@ -1129,7 +1129,7 @@ class RAGFlowPdfParser:
         for tag in re.findall(r"@@[0-9-]+\t[0-9.\t]+##", txt):
             pn, left, right, top, bottom = tag.strip("#").strip("@").split("\t")
             left, right, top, bottom = float(left), float(right), float(top), float(bottom)
-            poss.append(([int(p) - 1 for p in pn.split("-")], left, right, top, bottom))
+            poss.append(([int(p) - 1 for p in pn.split("-")], int(left), int(right), int(top), int(bottom)))
         return poss
 
     def crop(self, text, ZM=3, need_position=False):
@@ -1274,16 +1274,12 @@ class VisionParser(RAGFlowPdfParser):
                 prompt=vision_llm_describe_prompt(page=pdf_page_num + 1),
                 callback=callback,
             )
-
             if kwargs.get("callback"):
                 kwargs["callback"](idx * 1.0 / len(self.page_images), f"Processed: {idx + 1}/{len(self.page_images)}")
 
             if text:
                 width, height = self.page_images[idx].size
-                all_docs.append((
-                    text,
-                    f"@@{pdf_page_num + 1}\t{0.0:.1f}\t{width / zoomin:.1f}\t{0.0:.1f}\t{height / zoomin:.1f}##"
-                ))
+                all_docs.append((text, f"{pdf_page_num + 1} 0 {width / zoomin} 0 {height / zoomin}"))
         return all_docs, []
 
 

deepdoc/vision/ocr.py

@@ -84,8 +84,7 @@ def load_model(model_dir, nm, device_id: int | None = None):
     def cuda_is_available():
         try:
             import torch
-            target_id = 0 if device_id is None else device_id
-            if torch.cuda.is_available() and torch.cuda.device_count() > target_id:
+            if torch.cuda.is_available() and torch.cuda.device_count() > device_id:
                 return True
         except Exception:
             return False
@@ -101,13 +100,10 @@ def load_model(model_dir, nm, device_id: int | None = None):
     # Shrink GPU memory after execution
     run_options = ort.RunOptions()
     if cuda_is_available():
-        gpu_mem_limit_mb = int(os.environ.get("OCR_GPU_MEM_LIMIT_MB", "2048"))
-        arena_strategy = os.environ.get("OCR_ARENA_EXTEND_STRATEGY", "kNextPowerOfTwo")
-        provider_device_id = 0 if device_id is None else device_id
         cuda_provider_options = {
-            "device_id": provider_device_id, # Use specific GPU
-            "gpu_mem_limit": max(gpu_mem_limit_mb, 0) * 1024 * 1024,
-            "arena_extend_strategy": arena_strategy,  # gpu memory allocation strategy
+            "device_id": device_id, # Use specific GPU
+            "gpu_mem_limit": 512 * 1024 * 1024, # Limit gpu memory
+            "arena_extend_strategy": "kNextPowerOfTwo",  # gpu memory allocation strategy
         }
         sess = ort.InferenceSession(
             model_file_path,
@@ -115,8 +111,8 @@ def load_model(model_dir, nm, device_id: int | None = None):
             providers=['CUDAExecutionProvider'],
             provider_options=[cuda_provider_options]
             )
-        run_options.add_run_config_entry("memory.enable_memory_arena_shrinkage", "gpu:" + str(provider_device_id))
-        logging.info(f"load_model {model_file_path} uses GPU (device {provider_device_id}, gpu_mem_limit={cuda_provider_options['gpu_mem_limit']}, arena_strategy={arena_strategy})")
+        run_options.add_run_config_entry("memory.enable_memory_arena_shrinkage", "gpu:" + str(device_id))
+        logging.info(f"load_model {model_file_path} uses GPU")
     else:
         sess = ort.InferenceSession(
             model_file_path,

docker/docker-compose-base.yml

@@ -77,7 +77,7 @@ services:
     container_name: ragflow-infinity
     profiles:
       - infinity
-    image: infiniflow/infinity:v0.6.0-dev7
+    image: infiniflow/infinity:v0.6.0-dev5
     volumes:
       - infinity_data:/var/infinity
       - ./infinity_conf.toml:/infinity_conf.toml

docs/develop/mcp/launch_mcp_server.md

@@ -98,7 +98,7 @@ Where:
 
 - `mcp-host`: The MCP server's host address.
 - `mcp-port`: The MCP server's listening port.
-- `mcp-base-url`: The address of the running RAGFlow server.
+- `mcp-base_url`: The address of the running RAGFlow server.
 - `mcp-script-path`: The file path to the MCP server’s main script.
 - `mcp-mode`: The launch mode.
   - `self-host`: (default) self-host mode.

docs/guides/manage_users_and_services.md

@@ -1,360 +0,0 @@
-# Admin CLI and Admin Service
-
-
-
-The Admin CLI and Admin Service form a client-server architectural suite for RAGflow system administration. The Admin CLI serves as an interactive command-line interface that receives instructions and displays execution results from the Admin Service in real-time. This duo enables real-time monitoring of system operational status, supporting visibility into RAGflow Server services and dependent components including MySQL, Elasticsearch, Redis, and MinIO. In administrator mode, they provide user management capabilities that allow viewing users and performing critical operations—such as user creation, password updates, activation status changes, and comprehensive user data deletion—even when corresponding web interface functionalities are disabled.
-
-
-
-## Starting the Admin Service
-
-1. Before start Admin Service, please make sure RAGFlow system is already started.
-2. Switch to ragflow/ directory and run the service script:
-
-```bash
-source .venv/bin/activate
-export PYTHONPATH=$(pwd)
-python admin/admin_server.py
-```
-
-The service will start and listen for incoming connections from the CLI on the configured port. Default port is 9381.
-
-
-
-## Using the Admin CLI
-
-1. Ensure the Admin Service is running.
-2. Launch the CLI client:
-
-```bash
-source .venv/bin/activate
-export PYTHONPATH=$(pwd)
-python admin/admin_client.py -h 0.0.0.0 -p 9381
-```
-
-Enter superuser's password to login. Default password is `admin`.
-
-
-
-## Supported Commands
-
-Commands are case-insensitive and must be terminated with a semicolon(;).
-
-### Service manage commands
-
- `LIST SERVICES;`
-
-- Lists all available services within the RAGFLow system.
-
-- [Example](#example-list-services)
-
-`SHOW SERVICE <id>;`
-
-- Shows detailed status information for the service identified by <id>.
-- [Example](#example-show-service)
-
-### User Management Commands
-
-`LIST USERS;`
-
-- Lists all users known to the system.
-- [Example](#example-list-users)
-
-`SHOW USER <username>;`
-
-- Shows details and permissions for the user specified by **email**. The username must be enclosed in single or double quotes.
-- [Example](#example-show-user)
-
-`CREATE USER <username> <password>;`
-
-- Create user by username and password. The username and password must be enclosed in single or double quotes.
-- [Example](#example-create-user)
-
-`DROP USER <username>;`
-
-- Removes the specified user from the system. Use with caution.
-- [Example](#example-drop-user)
-
-`ALTER USER PASSWORD <username> <new_password>;`
-
-- Changes the password for the specified user.
-- [Example](#example-alter-user-password)
-
-`ALTER USER ACTIVE <username> <on/off>;`
-
-- Changes the user to active or inactive.
-- [Example](#example-alter-user-active)
-
-### Data and Agent Commands
-
-`LIST DATASETS OF <username>;`
-
-- Lists the datasets associated with the specified user.
-- [Example](#example-list-datasets-of-user)
-
-`LIST AGENTS OF <username>;`
-
-- Lists the agents associated with the specified user.
-- [Example](#example-list-agents-of-user)
-
-### Meta-Commands
-
-- \? or \help
-  Shows help information for the available commands.
-- \q or \quit
-  Exits the CLI application.
-- [Example](#example-meta-commands)
-
-### Examples
-
-<span id="example-list-services"></span>
-
-- List all available services.
-
-```
-admin> list services;
-command: list services;
-Listing all services
-+-------------------------------------------------------------------------------------------+-----------+----+---------------+-------+----------------+
-| extra                                                                                     | host      | id | name          | port  | service_type   |
-+-------------------------------------------------------------------------------------------+-----------+----+---------------+-------+----------------+
-| {}                                                                                        | 0.0.0.0   | 0  | ragflow_0     | 9380  | ragflow_server |
-| {'meta_type': 'mysql', 'password': 'infini_rag_flow', 'username': 'root'}                 | localhost | 1  | mysql         | 5455  | meta_data      |
-| {'password': 'infini_rag_flow', 'store_type': 'minio', 'user': 'rag_flow'}                | localhost | 2  | minio         | 9000  | file_store     |
-| {'password': 'infini_rag_flow', 'retrieval_type': 'elasticsearch', 'username': 'elastic'} | localhost | 3  | elasticsearch | 1200  | retrieval      |
-| {'db_name': 'default_db', 'retrieval_type': 'infinity'}                                   | localhost | 4  | infinity      | 23817 | retrieval      |
-| {'database': 1, 'mq_type': 'redis', 'password': 'infini_rag_flow'}                        | localhost | 5  | redis         | 6379  | message_queue  |
-+-------------------------------------------------------------------------------------------+-----------+----+---------------+-------+----------------+
-
-```
-
-<span id="example-show-service"></span>
-
-- Show ragflow_server.
-
-```
-admin> show service 0;
-command: show service 0;
-Showing service: 0
-Service ragflow_0 is alive. Detail:
-Confirm elapsed: 26.0 ms.
-```
-
-- Show mysql.
-
-```
-admin> show service 1;
-command: show service 1;
-Showing service: 1
-Service mysql is alive. Detail:
-+---------+----------+------------------+------+------------------+------------------------+-------+-----------------+
-| command | db       | host             | id   | info             | state                  | time  | user            |
-+---------+----------+------------------+------+------------------+------------------------+-------+-----------------+
-| Daemon  | None     | localhost        | 5    | None             | Waiting on empty queue | 16111 | event_scheduler |
-| Sleep   | rag_flow | 172.18.0.1:40046 | 1610 | None             |                        | 2     | root            |
-| Query   | rag_flow | 172.18.0.1:35882 | 1629 | SHOW PROCESSLIST | init                   | 0     | root            |
-+---------+----------+------------------+------+------------------+------------------------+-------+-----------------+
-```
-
-- Show minio.
-
-```
-admin> show service 2;
-command: show service 2;
-Showing service: 2
-Service minio is alive. Detail:
-Confirm elapsed: 2.1 ms.
-```
-
-- Show elasticsearch.
-
-```
-admin> show service 3;
-command: show service 3;
-Showing service: 3
-Service elasticsearch is alive. Detail:
-+----------------+------+--------------+---------+----------------+--------------+---------------+--------------+------------------------------+----------------------------+-----------------+-------+---------------+---------+-------------+---------------------+--------+------------+--------------------+
-| cluster_name   | docs | docs_deleted | indices | indices_shards | jvm_heap_max | jvm_heap_used | jvm_versions | mappings_deduplicated_fields | mappings_deduplicated_size | mappings_fields | nodes | nodes_version | os_mem  | os_mem_used | os_mem_used_percent | status | store_size | total_dataset_size |
-+----------------+------+--------------+---------+----------------+--------------+---------------+--------------+------------------------------+----------------------------+-----------------+-------+---------------+---------+-------------+---------------------+--------+------------+--------------------+
-| docker-cluster | 717  | 86           | 37      | 42             | 3.76 GB      | 1.74 GB       | 21.0.1+12-29 | 6575                         | 48.0 KB                    | 8521            | 1     | ['8.11.3']    | 7.52 GB | 4.55 GB     | 61                  | green  | 4.60 MB    | 4.60 MB            |
-+----------------+------+--------------+---------+----------------+--------------+---------------+--------------+------------------------------+----------------------------+-----------------+-------+---------------+---------+-------------+---------------------+--------+------------+--------------------+
-```
-
-- Show infinity.
-
-```
-admin> show service 4;
-command: show service 4;
-Showing service: 4
-Fail to show service, code: 500, message: Infinity is not in use.
-```
-
-- Show redis.
-
-```
-admin> show service 5;
-command: show service 5;
-Showing service: 5
-Service redis is alive. Detail:
-+-----------------+-------------------+---------------------------+-------------------------+---------------+-------------+--------------------------+---------------------+-------------+
-| blocked_clients | connected_clients | instantaneous_ops_per_sec | mem_fragmentation_ratio | redis_version | server_mode | total_commands_processed | total_system_memory | used_memory |
-+-----------------+-------------------+---------------------------+-------------------------+---------------+-------------+--------------------------+---------------------+-------------+
-| 0               | 2                 | 1                         | 10.41                   | 7.2.4         | standalone  | 10446                    | 30.84G              | 1.10M       |
-+-----------------+-------------------+---------------------------+-------------------------+---------------+-------------+--------------------------+---------------------+-------------+
-```
-
-<span id="example-list-users"></span>
-
-- List all user.
-
-```
-admin> list users;
-command: list users;
-Listing all users
-+-------------------------------+----------------------+-----------+----------+
-| create_date                   | email                | is_active | nickname |
-+-------------------------------+----------------------+-----------+----------+
-| Mon, 22 Sep 2025 10:59:04 GMT | admin@ragflow.io     | 1         | admin    |
-| Sun, 14 Sep 2025 17:36:27 GMT | lynn_inf@hotmail.com | 1         | Lynn     |
-+-------------------------------+----------------------+-----------+----------+
-```
-
-<span id="example-show-user"></span>
-
-- Show specified user.
-
-```
-admin> show user "admin@ragflow.io";
-command: show user "admin@ragflow.io";
-Showing user: admin@ragflow.io
-+-------------------------------+------------------+-----------+--------------+------------------+--------------+----------+-----------------+---------------+--------+-------------------------------+
-| create_date                   | email            | is_active | is_anonymous | is_authenticated | is_superuser | language | last_login_time | login_channel | status | update_date                   |
-+-------------------------------+------------------+-----------+--------------+------------------+--------------+----------+-----------------+---------------+--------+-------------------------------+
-| Mon, 22 Sep 2025 10:59:04 GMT | admin@ragflow.io | 1         | 0            | 1                | True         | Chinese  | None            | None          | 1      | Mon, 22 Sep 2025 10:59:04 GMT |
-+-------------------------------+------------------+-----------+--------------+------------------+--------------+----------+-----------------+---------------+--------+-------------------------------+
-```
-
-<span id="example-create-user"></span>
-
-- Create new user.
-
-```
-admin> create user "example@ragflow.io" "psw";
-command: create user "example@ragflow.io" "psw";
-Create user: example@ragflow.io, password: psw, role: user
-+----------------------------------+--------------------+----------------------------------+--------------+---------------+----------+
-| access_token                     | email              | id                               | is_superuser | login_channel | nickname |
-+----------------------------------+--------------------+----------------------------------+--------------+---------------+----------+
-| 5cdc6d1e9df111f099b543aee592c6bf | example@ragflow.io | 5cdc6ca69df111f099b543aee592c6bf | False        | password      |          |
-+----------------------------------+--------------------+----------------------------------+--------------+---------------+----------+
-```
-
-<span id="example-alter-user-password"></span>
-
-- Alter user password.
-
-```
-admin> alter user password "example@ragflow.io" "newpsw";
-command: alter user password "example@ragflow.io" "newpsw";
-Alter user: example@ragflow.io, password: newpsw
-Password updated successfully!
-```
-
-<span id="example-alter-user-active"></span>
-
-- Alter user active, turn off.
-
-```
-admin> alter user active "example@ragflow.io" off;
-command: alter user active "example@ragflow.io" off;
-Alter user example@ragflow.io activate status, turn off.
-Turn off user activate status successfully!
-```
-
-<span id="example-drop-user"></span>
-
-- Drop user.
-
-```
-admin> Drop user "example@ragflow.io";
-command: Drop user "example@ragflow.io";
-Drop user: example@ragflow.io
-Successfully deleted user. Details:
-Start to delete owned tenant.
-- Deleted 2 tenant-LLM records.
-- Deleted 0 langfuse records.
-- Deleted 1 tenant.
-- Deleted 1 user-tenant records.
-- Deleted 1 user.
-Delete done!
-```
-
-Delete user's data at the same time.
-
-<span id="example-list-datasets-of-user"></span>
-
-- List the specified user's dataset.
-
-```
-admin> list datasets of "lynn_inf@hotmail.com";
-command: list datasets of "lynn_inf@hotmail.com";
-Listing all datasets of user: lynn_inf@hotmail.com
-+-----------+-------------------------------+---------+----------+---------------+------------+--------+-----------+-------------------------------+
-| chunk_num | create_date                   | doc_num | language | name          | permission | status | token_num | update_date                   |
-+-----------+-------------------------------+---------+----------+---------------+------------+--------+-----------+-------------------------------+
-| 29        | Mon, 15 Sep 2025 11:56:59 GMT | 12      | Chinese  | test_dataset  | me         | 1      | 12896     | Fri, 19 Sep 2025 17:50:58 GMT |
-| 4         | Sun, 28 Sep 2025 11:49:31 GMT | 6       | Chinese  | dataset_share | team       | 1      | 1121      | Sun, 28 Sep 2025 14:41:03 GMT |
-+-----------+-------------------------------+---------+----------+---------------+------------+--------+-----------+-------------------------------+
-```
-
-<span id="example-list-agents-of-user"></span>
-
-- List the specified user's agents.
-
-```
-admin> list agents of "lynn_inf@hotmail.com";
-command: list agents of "lynn_inf@hotmail.com";
-Listing all agents of user: lynn_inf@hotmail.com
-+-----------------+-------------+------------+-----------------+
-| canvas_category | canvas_type | permission | title           |
-+-----------------+-------------+------------+-----------------+
-| agent_canvas    | None        | team       | research_helper |
-+-----------------+-------------+------------+-----------------+
-```
-
-<span id="example-meta-commands"></span>
-
-- Show help infomation.
-
-```
-admin> \help
-command: \help
-
-Commands:
-  LIST SERVICES
-  SHOW SERVICE <service>
-  STARTUP SERVICE <service>
-  SHUTDOWN SERVICE <service>
-  RESTART SERVICE <service>
-  LIST USERS
-  SHOW USER <user>
-  DROP USER <user>
-  CREATE USER <user> <password>
-  ALTER USER PASSWORD <user> <new_password>
-  ALTER USER ACTIVE <user> <on/off>
-  LIST DATASETS OF <user>
-  LIST AGENTS OF <user>
-
-Meta Commands:
-  \?, \h, \help     Show this help
-  \q, \quit, \exit   Quit the CLI
-```
-
-- Exit
-
-```
-admin> \q
-command: \q
-Goodbye!
-```
-

docs/references/http_api_reference.md

@@ -830,8 +830,7 @@ Success:
             "update_time": 1728533243536,
             "vector_similarity_weight": 0.3
         }
-    ],
-    "total": 1
+    ]
 }
 ```
 
@@ -1281,7 +1280,7 @@ Success:
                 "update_time": 1728897061948
             }
         ],
-        "total_datasets": 1
+        "total": 1
     }
 }
 ```

docs/references/supported_models.mdx

@@ -66,7 +66,6 @@ A complete list of models supported by RAGFlow, which will continue to expand.
 | DeepInfra             | :heavy_check_mark: | :heavy_check_mark: |                    |                    | :heavy_check_mark: | :heavy_check_mark: |
 | 302.AI                | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |                    |                    |
 | CometAPI              | :heavy_check_mark: | :heavy_check_mark: |                    |                    |                    |                    |
-| DeerAPI               | :heavy_check_mark: | :heavy_check_mark: |                    | :heavy_check_mark: |                    | :heavy_check_mark: |
 
 ```mdx-code-block
 </APITable>

download_deps.py

@@ -6,6 +6,7 @@
 # dependencies = [
 #   "huggingface-hub",
 #   "nltk",
+#   "argparse",
 # ]
 # ///
 

graphrag/general/index.py

@@ -55,7 +55,7 @@ async def run_graphrag(
     start = trio.current_time()
     tenant_id, kb_id, doc_id = row["tenant_id"], str(row["kb_id"]), row["doc_id"]
     chunks = []
-    for d in settings.retriever.chunk_list(doc_id, tenant_id, [kb_id], fields=["content_with_weight", "doc_id"], sort_by_position=True):
+    for d in settings.retrievaler.chunk_list(doc_id, tenant_id, [kb_id], fields=["content_with_weight", "doc_id"], sort_by_position=True):
         chunks.append(d["content_with_weight"])
 
     with trio.fail_after(max(120, len(chunks) * 60 * 10) if enable_timeout_assertion else 10000000000):
@@ -170,7 +170,7 @@ async def run_graphrag_for_kb(
         chunks = []
         current_chunk = ""
 
-        for d in settings.retriever.chunk_list(
+        for d in settings.retrievaler.chunk_list(
             doc_id,
             tenant_id,
             [kb_id],

graphrag/general/smoke.py

@@ -62,7 +62,7 @@ async def main():
 
     chunks = [
         d["content_with_weight"]
-        for d in settings.retriever.chunk_list(
+        for d in settings.retrievaler.chunk_list(
             args.doc_id,
             args.tenant_id,
             [kb_id],

graphrag/light/smoke.py

@@ -63,7 +63,7 @@ async def main():
 
     chunks = [
         d["content_with_weight"]
-        for d in settings.retriever.chunk_list(
+        for d in settings.retrievaler.chunk_list(
             args.doc_id,
             args.tenant_id,
             [kb_id],

graphrag/utils.py

@@ -341,7 +341,7 @@ def get_relation(tenant_id, kb_id, from_ent_name, to_ent_name, size=1):
     ents = list(set(ents))
     conds = {"fields": ["content_with_weight"], "size": size, "from_entity_kwd": ents, "to_entity_kwd": ents, "knowledge_graph_kwd": ["relation"]}
     res = []
-    es_res = settings.retriever.search(conds, search.index_name(tenant_id), [kb_id] if isinstance(kb_id, str) else kb_id)
+    es_res = settings.retrievaler.search(conds, search.index_name(tenant_id), [kb_id] if isinstance(kb_id, str) else kb_id)
     for id in es_res.ids:
         try:
             if size == 1:
@@ -398,7 +398,7 @@ async def does_graph_contains(tenant_id, kb_id, doc_id):
 
 async def get_graph_doc_ids(tenant_id, kb_id) -> list[str]:
     conds = {"fields": ["source_id"], "removed_kwd": "N", "size": 1, "knowledge_graph_kwd": ["graph"]}
-    res = await trio.to_thread.run_sync(lambda: settings.retriever.search(conds, search.index_name(tenant_id), [kb_id]))
+    res = await trio.to_thread.run_sync(lambda: settings.retrievaler.search(conds, search.index_name(tenant_id), [kb_id]))
     doc_ids = []
     if res.total == 0:
         return doc_ids
@@ -409,7 +409,7 @@ async def get_graph_doc_ids(tenant_id, kb_id) -> list[str]:
 
 async def get_graph(tenant_id, kb_id, exclude_rebuild=None):
     conds = {"fields": ["content_with_weight", "removed_kwd", "source_id"], "size": 1, "knowledge_graph_kwd": ["graph"]}
-    res = await trio.to_thread.run_sync(settings.retriever.search, conds, search.index_name(tenant_id), [kb_id])
+    res = await trio.to_thread.run_sync(settings.retrievaler.search, conds, search.index_name(tenant_id), [kb_id])
     if not res.total == 0:
         for id in res.ids:
             try:
@@ -562,7 +562,7 @@ def merge_tuples(list1, list2):
 
 
 async def get_entity_type2samples(idxnms, kb_ids: list):
-    es_res = await trio.to_thread.run_sync(lambda: settings.retriever.search({"knowledge_graph_kwd": "ty2ents", "kb_id": kb_ids, "size": 10000, "fields": ["content_with_weight"]}, idxnms, kb_ids))
+    es_res = await trio.to_thread.run_sync(lambda: settings.retrievaler.search({"knowledge_graph_kwd": "ty2ents", "kb_id": kb_ids, "size": 10000, "fields": ["content_with_weight"]}, idxnms, kb_ids))
 
     res = defaultdict(list)
     for id in es_res.ids:

helm/values.yaml

@@ -96,7 +96,7 @@ ragflow:
 infinity:
   image:
     repository: infiniflow/infinity
-    tag: v0.6.0-dev7
+    tag: v0.6.0-dev5
     pullPolicy: IfNotPresent
     pullSecrets: []
   storage:

pyproject.toml

@@ -46,7 +46,7 @@ dependencies = [
     "html-text==0.6.2",
     "httpx[socks]==0.27.2",
     "huggingface-hub>=0.25.0,<0.26.0",
-    "infinity-sdk==0.6.0.dev7",
+    "infinity-sdk==0.6.0.dev5",
     "infinity-emb>=0.0.66,<0.0.67",
     "itsdangerous==2.1.2",
     "json-repair==0.35.0",
@@ -119,7 +119,7 @@ dependencies = [
     "graspologic>=3.4.1,<4.0.0",
     "mini-racer>=0.12.4,<0.13.0",
     "pyodbc>=5.2.0,<6.0.0",
-    "pyicu>=2.15.3,<3.0.0",
+    "pyicu>=2.13.1,<3.0.0",
     "flasgger>=0.9.7.1,<0.10.0",
     "xxhash>=3.5.0,<4.0.0",
     "trio>=0.29.0",
@@ -133,8 +133,6 @@ dependencies = [
     "litellm>=1.74.15.post1",
     "flask-mail>=0.10.0",
     "lark>=1.2.2",
-    "mammoth>=1.11.0",
-    "markdownify>=1.2.0",
 ]
 
 [project.optional-dependencies]
@@ -161,7 +159,7 @@ test = [
 ]
 
 [[tool.uv.index]]
-url = "https://pypi.tuna.tsinghua.edu.cn/simple"
+url = "https://mirrors.aliyun.com/pypi/simple"
 
 [tool.setuptools]
 packages = [

rag/app/naive.py

@@ -256,49 +256,6 @@ class Docx(DocxParser):
             tbls.append(((None, html), ""))
         return new_line, tbls
 
-    def to_markdown(self, filename=None, binary=None, inline_images: bool = True):
-        """
-        This function uses mammoth, licensed under the BSD 2-Clause License.
-        """
-
-        import base64
-        import uuid
-
-        import mammoth
-        from markdownify import markdownify
-
-        docx_file = BytesIO(binary) if binary else open(filename, "rb")
-
-        def _convert_image_to_base64(image):
-            try:
-                with image.open() as image_file:
-                    image_bytes = image_file.read()
-                encoded = base64.b64encode(image_bytes).decode("utf-8")
-                base64_url = f"data:{image.content_type};base64,{encoded}"
-
-                alt_name = "image"
-                alt_name = f"img_{uuid.uuid4().hex[:8]}"
-
-                return {"src": base64_url, "alt": alt_name}
-            except Exception as e:
-                logging.warning(f"Failed to convert image to base64: {e}")
-                return {"src": "", "alt": "image"}
-
-        try:
-            if inline_images:
-                result = mammoth.convert_to_html(docx_file, convert_image=mammoth.images.img_element(_convert_image_to_base64))
-            else:
-                result = mammoth.convert_to_html(docx_file)
-
-            html = result.value
-
-            markdown_text = markdownify(html)
-            return markdown_text
-
-        finally:
-            if not binary:
-                docx_file.close()
-
 
 class Pdf(PdfParser):
     def __init__(self):
@@ -555,7 +512,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
             callback(0.2, "Visual model detected. Attempting to enhance figure extraction...")
         except Exception:
             vision_model = None
-
+        
         if vision_model:
             # Process images for each section
             section_images = []

rag/app/tag.py

@@ -133,14 +133,14 @@ def label_question(question, kbs):
     if tag_kb_ids:
         all_tags = get_tags_from_cache(tag_kb_ids)
         if not all_tags:
-            all_tags = settings.retriever.all_tags_in_portion(kb.tenant_id, tag_kb_ids)
+            all_tags = settings.retrievaler.all_tags_in_portion(kb.tenant_id, tag_kb_ids)
             set_tags_to_cache(tags=all_tags, kb_ids=tag_kb_ids)
         else:
             all_tags = json.loads(all_tags)
         tag_kbs = KnowledgebaseService.get_by_ids(tag_kb_ids)
         if not tag_kbs:
             return tags
-        tags = settings.retriever.tag_query(question,
+        tags = settings.retrievaler.tag_query(question,
                                               list(set([kb.tenant_id for kb in tag_kbs])),
                                               tag_kb_ids,
                                               all_tags,

rag/benchmark.py

@@ -52,7 +52,7 @@ class Benchmark:
         run = defaultdict(dict)
         query_list = list(qrels.keys())
         for query in query_list:
-            ranks = settings.retriever.retrieval(query, self.embd_mdl, self.tenant_id, [self.kb.id], 1, 30,
+            ranks = settings.retrievaler.retrieval(query, self.embd_mdl, self.tenant_id, [self.kb.id], 1, 30,
                                             0.0, self.vector_similarity_weight)
             if len(ranks["chunks"]) == 0:
                 print(f"deleted query: {query}")

rag/flow/chunker/__init__.py

@@ -12,27 +12,4 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-#
 
-
-class AdminException(Exception):
-    def __init__(self, message, code=400):
-        super().__init__(message)
-        self.type = "admin"
-        self.code = code
-        self.message = message
-
-
-class UserNotFoundError(AdminException):
-    def __init__(self, username):
-        super().__init__(f"User '{username}' not found", 404)
-
-
-class UserAlreadyExistsError(AdminException):
-    def __init__(self, username):
-        super().__init__(f"User '{username}' already exists", 409)
-
-
-class CannotDeleteAdminError(AdminException):
-    def __init__(self):
-        super().__init__("Cannot delete admin account", 403)

rag/flow/chunker/chunker.py

@@ -0,0 +1,299 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import json
+import random
+import trio
+from api.db import LLMType
+from api.db.services.llm_service import LLMBundle
+from deepdoc.parser.pdf_parser import RAGFlowPdfParser
+from graphrag.utils import chat_limiter, get_llm_cache, set_llm_cache
+from rag.flow.base import ProcessBase, ProcessParamBase
+from rag.flow.chunker.schema import ChunkerFromUpstream
+from rag.nlp import naive_merge, naive_merge_with_images, concat_img
+from rag.prompts.prompts import keyword_extraction, question_proposal, detect_table_of_contents, \
+    table_of_contents_index, toc_transformer
+from rag.utils import num_tokens_from_string
+
+
+class ChunkerParam(ProcessParamBase):
+    def __init__(self):
+        super().__init__()
+        self.method_options = [
+            # General
+            "general",
+            "onetable",
+            # Customer Service
+            "q&a",
+            "manual",
+            # Recruitment
+            "resume",
+            # Education & Research
+            "book",
+            "paper",
+            "laws",
+            "presentation",
+            "toc" # table of contents
+            # Other
+            # "Tag" # TODO: Other method
+        ]
+        self.method = "general"
+        self.chunk_token_size = 512
+        self.delimiter = "\n"
+        self.overlapped_percent = 0
+        self.page_rank = 0
+        self.auto_keywords = 0
+        self.auto_questions = 0
+        self.tag_sets = []
+        self.llm_setting = {"llm_id": "", "lang": "Chinese"}
+
+    def check(self):
+        self.check_valid_value(self.method.lower(), "Chunk method abnormal.", self.method_options)
+        self.check_positive_integer(self.chunk_token_size, "Chunk token size.")
+        self.check_nonnegative_number(self.page_rank, "Page rank value: (0, 10]")
+        self.check_nonnegative_number(self.auto_keywords, "Auto-keyword value: (0, 10]")
+        self.check_nonnegative_number(self.auto_questions, "Auto-question value: (0, 10]")
+        self.check_decimal_float(self.overlapped_percent, "Overlapped percentage: [0, 1)")
+
+    def get_input_form(self) -> dict[str, dict]:
+        return {}
+
+
+class Chunker(ProcessBase):
+    component_name = "Chunker"
+
+    def _general(self, from_upstream: ChunkerFromUpstream):
+        self.callback(random.randint(1, 5) / 100.0, "Start to chunk via `General`.")
+        if from_upstream.output_format in ["markdown", "text", "html"]:
+            if from_upstream.output_format == "markdown":
+                payload = from_upstream.markdown_result
+            elif from_upstream.output_format == "text":
+                payload = from_upstream.text_result
+            else:  # == "html"
+                payload = from_upstream.html_result
+
+            if not payload:
+                payload = ""
+
+            cks = naive_merge(
+                payload,
+                self._param.chunk_token_size,
+                self._param.delimiter,
+                self._param.overlapped_percent,
+            )
+            return [{"text": c} for c in cks]
+
+        # json
+        sections, section_images = [], []
+        for o in from_upstream.json_result or []:
+            sections.append((o.get("text", ""), o.get("position_tag", "")))
+            section_images.append(o.get("image"))
+
+        chunks, images = naive_merge_with_images(
+            sections,
+            section_images,
+            self._param.chunk_token_size,
+            self._param.delimiter,
+            self._param.overlapped_percent,
+        )
+
+        return [
+            {
+                "text": RAGFlowPdfParser.remove_tag(c),
+                "image": img,
+                "positions": RAGFlowPdfParser.extract_positions(c),
+            }
+            for c, img in zip(chunks, images)
+        ]
+
+    def _q_and_a(self, from_upstream: ChunkerFromUpstream):
+        pass
+
+    def _resume(self, from_upstream: ChunkerFromUpstream):
+        pass
+
+    def _manual(self, from_upstream: ChunkerFromUpstream):
+        pass
+
+    def _table(self, from_upstream: ChunkerFromUpstream):
+        pass
+
+    def _paper(self, from_upstream: ChunkerFromUpstream):
+        pass
+
+    def _book(self, from_upstream: ChunkerFromUpstream):
+        pass
+
+    def _laws(self, from_upstream: ChunkerFromUpstream):
+        pass
+
+    def _presentation(self, from_upstream: ChunkerFromUpstream):
+        pass
+
+    def _one(self, from_upstream: ChunkerFromUpstream):
+        pass
+
+    def _toc(self, from_upstream: ChunkerFromUpstream):
+        self.callback(random.randint(1, 5) / 100.0, "Start to chunk via `ToC`.")
+        if from_upstream.output_format in ["markdown", "text", "html"]:
+            return
+
+        # json
+        sections, section_images, page_1024, tc_arr = [], [], [""], [0]
+        for o in from_upstream.json_result or []:
+            txt = o.get("text", "")
+            tc = num_tokens_from_string(txt)
+            page_1024[-1] += "\n" + txt
+            tc_arr[-1] += tc
+            if tc_arr[-1] > 1024:
+                page_1024.append("")
+                tc_arr.append(0)
+            sections.append((o.get("text", ""), o.get("position_tag", "")))
+            section_images.append(o.get("image"))
+            print(len(sections), o)
+
+        llm_setting = self._param.llm_setting
+        chat_mdl = LLMBundle(self._canvas._tenant_id, LLMType.CHAT, llm_name=llm_setting["llm_id"], lang=llm_setting["lang"])
+        self.callback(random.randint(5, 15) / 100.0, "Start to detect table of contents...")
+        toc_secs = detect_table_of_contents(page_1024, chat_mdl)
+        if toc_secs:
+            self.callback(random.randint(25, 35) / 100.0, "Start to extract table of contents...")
+            toc_arr = toc_transformer(toc_secs, chat_mdl)
+            toc_arr = [it for it in toc_arr if it.get("structure")]
+            print(json.dumps(toc_arr, ensure_ascii=False, indent=2), flush=True)
+            self.callback(random.randint(35, 75) / 100.0, "Start to link table of contents...")
+            toc_arr = table_of_contents_index(toc_arr, [t for t,_ in sections], chat_mdl)
+            for i in range(len(toc_arr)-1):
+                if not toc_arr[i].get("indices"):
+                    continue
+
+                for j in range(i+1, len(toc_arr)):
+                    if toc_arr[j].get("indices"):
+                        if toc_arr[j]["indices"][0] - toc_arr[i]["indices"][-1] > 1:
+                            toc_arr[i]["indices"].extend([x for x in range(toc_arr[i]["indices"][-1]+1, toc_arr[j]["indices"][0])])
+                        break
+            # put all sections ahead of toc_arr[0] into it
+            # for i in range(len(toc_arr)):
+            #     if toc_arr[i].get("indices") and toc_arr[i]["indices"][0]:
+            #         toc_arr[i]["indices"] = [x for x in range(toc_arr[i]["indices"][-1]+1)]
+            #         break
+            # put all sections after toc_arr[-1] into it
+            for i in range(len(toc_arr)-1, -1, -1):
+                if toc_arr[i].get("indices") and toc_arr[i]["indices"][-1]:
+                    toc_arr[i]["indices"] = [x for x in range(toc_arr[i]["indices"][0], len(sections))]
+                    break
+            print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n", json.dumps(toc_arr, ensure_ascii=False, indent=2), flush=True)
+
+            chunks, images = [], []
+            for it in toc_arr:
+                if not it.get("indices"):
+                    continue
+                txt = ""
+                img = None
+                for i in it["indices"]:
+                    idx = i
+                    txt += "\n" + sections[idx][0] + "\t" + sections[idx][1]
+                    if img and section_images[idx]:
+                        img = concat_img(img, section_images[idx])
+                    elif section_images[idx]:
+                        img = section_images[idx]
+
+                it["indices"] = []
+                if not txt:
+                    continue
+                it["indices"] = [len(chunks)]
+                print(it, "KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKK\n", txt)
+                chunks.append(txt)
+                images.append(img)
+            self.callback(1, "Done")
+            return [
+                {
+                    "text": RAGFlowPdfParser.remove_tag(c),
+                    "image": img,
+                    "positions": RAGFlowPdfParser.extract_positions(c),
+                }
+                for c, img in zip(chunks, images)
+            ]
+
+        self.callback(message="No table of contents detected.")
+
+
+    async def _invoke(self, **kwargs):
+        function_map = {
+            "general": self._general,
+            "q&a": self._q_and_a,
+            "resume": self._resume,
+            "manual": self._manual,
+            "table": self._table,
+            "paper": self._paper,
+            "book": self._book,
+            "laws": self._laws,
+            "presentation": self._presentation,
+            "one": self._one,
+        }
+
+        try:
+            from_upstream = ChunkerFromUpstream.model_validate(kwargs)
+        except Exception as e:
+            self.set_output("_ERROR", f"Input error: {str(e)}")
+            return
+
+        chunks = function_map[self._param.method](from_upstream)
+        llm_setting = self._param.llm_setting
+
+        async def auto_keywords():
+            nonlocal chunks, llm_setting
+            chat_mdl = LLMBundle(self._canvas._tenant_id, LLMType.CHAT, llm_name=llm_setting["llm_id"], lang=llm_setting["lang"])
+
+            async def doc_keyword_extraction(chat_mdl, ck, topn):
+                cached = get_llm_cache(chat_mdl.llm_name, ck["text"], "keywords", {"topn": topn})
+                if not cached:
+                    async with chat_limiter:
+                        cached = await trio.to_thread.run_sync(lambda: keyword_extraction(chat_mdl, ck["text"], topn))
+                    set_llm_cache(chat_mdl.llm_name, ck["text"], cached, "keywords", {"topn": topn})
+                if cached:
+                    ck["keywords"] = cached.split(",")
+
+            async with trio.open_nursery() as nursery:
+                for ck in chunks:
+                    nursery.start_soon(doc_keyword_extraction, chat_mdl, ck, self._param.auto_keywords)
+
+        async def auto_questions():
+            nonlocal chunks, llm_setting
+            chat_mdl = LLMBundle(self._canvas._tenant_id, LLMType.CHAT, llm_name=llm_setting["llm_id"], lang=llm_setting["lang"])
+
+            async def doc_question_proposal(chat_mdl, d, topn):
+                cached = get_llm_cache(chat_mdl.llm_name, ck["text"], "question", {"topn": topn})
+                if not cached:
+                    async with chat_limiter:
+                        cached = await trio.to_thread.run_sync(lambda: question_proposal(chat_mdl, ck["text"], topn))
+                    set_llm_cache(chat_mdl.llm_name, ck["text"], cached, "question", {"topn": topn})
+                if cached:
+                    d["questions"] = cached.split("\n")
+
+            async with trio.open_nursery() as nursery:
+                for ck in chunks:
+                    nursery.start_soon(doc_question_proposal, chat_mdl, ck, self._param.auto_questions)
+
+        async with trio.open_nursery() as nursery:
+            if self._param.auto_questions:
+                nursery.start_soon(auto_questions)
+            if self._param.auto_keywords:
+                nursery.start_soon(auto_keywords)
+
+        if self._param.page_rank:
+            for ck in chunks:
+                ck["page_rank"] = self._param.page_rank
+
+        self.set_output("chunks", chunks)

rag/flow/chunker/schema.py

@@ -0,0 +1,37 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ChunkerFromUpstream(BaseModel):
+    created_time: float | None = Field(default=None, alias="_created_time")
+    elapsed_time: float | None = Field(default=None, alias="_elapsed_time")
+
+    name: str
+    file: dict | None = Field(default=None)
+
+    output_format: Literal["json", "markdown", "text", "html"] | None = Field(default=None)
+
+    json_result: list[dict[str, Any]] | None = Field(default=None, alias="json")
+    markdown_result: str | None = Field(default=None, alias="markdown")
+    text_result: str | None = Field(default=None, alias="text")
+    html_result: list[str] | None = Field(default=None, alias="html")
+
+    model_config = ConfigDict(populate_by_name=True, extra="forbid")
+
+    # def to_dict(self, *, exclude_none: bool = True) -> dict:
+    #     return self.model_dump(by_alias=True, exclude_none=exclude_none)

rag/flow/parser/parser.py

@@ -52,7 +52,6 @@ class ParserParam(ProcessParamBase):
             ],
             "word": [
                 "json",
-                "markdown",
             ],
             "slides": [
                 "json",
@@ -248,15 +247,13 @@ class Parser(ProcessBase):
         conf = self._param.setups["word"]
         self.set_output("output_format", conf["output_format"])
         docx_parser = Docx()
-
+        sections, tbls = docx_parser(name, binary=blob)
+        sections = [{"text": section[0], "image": section[1]} for section in sections if section]
+        sections.extend([{"text": tb, "image": None} for ((_,tb), _) in tbls])
+        # json
+        assert conf.get("output_format") == "json", "have to be json for doc"
         if conf.get("output_format") == "json":
-            sections, tbls = docx_parser(name, binary=blob)
-            sections = [{"text": section[0], "image": section[1]} for section in sections if section]
-            sections.extend([{"text": tb, "image": None} for ((_,tb), _) in tbls])
             self.set_output("json", sections)
-        elif conf.get("output_format") == "markdown":
-            markdown_text = docx_parser.to_markdown(name, binary=blob)
-            self.set_output("markdown", markdown_text)
 
     def _slides(self, name, blob):
         from deepdoc.parser.ppt_parser import RAGFlowPptParser as ppt_parser

rag/llm/chat_model.py

@@ -133,7 +133,6 @@ class Base(ABC):
             "logprobs",
             "top_logprobs",
             "extra_headers",
-            "enable_thinking"
         }
 
         gen_conf = {k: v for k, v in gen_conf.items() if k in allowed_conf}
@@ -1276,14 +1275,6 @@ class TokenPonyChat(Base):
         if not base_url:
             base_url = "https://ragflow.vip-api.tokenpony.cn/v1"
 
-class DeerAPIChat(Base):
-    _FACTORY_NAME = "DeerAPI"
-
-    def __init__(self, key, model_name, base_url="https://api.deerapi.com/v1", **kwargs):
-        if not base_url:
-            base_url = "https://api.deerapi.com/v1"
-        super().__init__(key, model_name, base_url, **kwargs)
-
 
 class LiteLLMBase(ABC):
     _FACTORY_NAME = [

rag/llm/cv_model.py

@@ -26,7 +26,7 @@ from openai.lib.azure import AzureOpenAI
 from zhipuai import ZhipuAI
 from rag.nlp import is_english
 from rag.prompts.generator import vision_llm_describe_prompt
-from rag.utils import num_tokens_from_string, total_token_count_from_response
+from rag.utils import num_tokens_from_string
 
 
 class Base(ABC):
@@ -125,7 +125,7 @@ class Base(ABC):
             b64 = base64.b64encode(data).decode("utf-8")
             return f"data:{mime};base64,{b64}"
         with BytesIO() as buffered:
-            fmt = "jpeg"
+            fmt = "JPEG"
             try:
                 image.save(buffered, format="JPEG")
             except Exception:
@@ -133,10 +133,10 @@ class Base(ABC):
                 buffered.seek(0)
                 buffered.truncate()
                 image.save(buffered, format="PNG")
-                fmt = "png"
+                fmt = "PNG"
             data = buffered.getvalue()
             b64 = base64.b64encode(data).decode("utf-8")
-            mime = f"image/{fmt}"
+            mime = f"image/{fmt.lower()}"
         return f"data:{mime};base64,{b64}"
 
     def prompt(self, b64):
@@ -178,7 +178,7 @@ class GptV4(Base):
             model=self.model_name,
             messages=self.prompt(b64),
         )
-        return res.choices[0].message.content.strip(), total_token_count_from_response(res)
+        return res.choices[0].message.content.strip(), res.usage.total_tokens
 
     def describe_with_prompt(self, image, prompt=None):
         b64 = self.image2base64(image)
@@ -186,7 +186,7 @@ class GptV4(Base):
             model=self.model_name,
             messages=self.vision_llm_prompt(b64, prompt),
         )
-        return res.choices[0].message.content.strip(),total_token_count_from_response(res)
+        return res.choices[0].message.content.strip(), res.usage.total_tokens
 
 
 class AzureGptV4(GptV4):
@@ -522,10 +522,11 @@ class GeminiCV(Base):
         )
         b64 = self.image2base64(image)
         with BytesIO(base64.b64decode(b64)) as bio:
-            with open(bio) as img:
-                input = [prompt, img]
-                res = self.model.generate_content(input)
-                return res.text, total_token_count_from_response(res)
+            img = open(bio)
+            input = [prompt, img]
+            res = self.model.generate_content(input)
+            img.close()
+            return res.text, res.usage_metadata.total_token_count
 
     def describe_with_prompt(self, image, prompt=None):
         from PIL.Image import open
@@ -533,10 +534,11 @@ class GeminiCV(Base):
         b64 = self.image2base64(image)
         vision_prompt = prompt if prompt else vision_llm_describe_prompt()
         with BytesIO(base64.b64decode(b64)) as bio:
-            with open(bio) as img:
-                input = [vision_prompt, img]
-                res = self.model.generate_content(input)
-                return res.text, total_token_count_from_response(res)
+            img = open(bio)
+            input = [vision_prompt, img]
+            res = self.model.generate_content(input)
+            img.close()
+            return res.text, res.usage_metadata.total_token_count
 
     def chat(self, system, history, gen_conf, images=[]):
         generation_config = dict(temperature=gen_conf.get("temperature", 0.3), top_p=gen_conf.get("top_p", 0.7))
@@ -545,7 +547,7 @@ class GeminiCV(Base):
                 self._form_history(system, history, images),
                 generation_config=generation_config)
             ans = response.text
-            return ans, total_token_count_from_response(ans)
+            return ans, response.usage_metadata.total_token_count
         except Exception as e:
             return "**ERROR**: " + str(e), 0
 
@@ -568,7 +570,10 @@ class GeminiCV(Base):
         except Exception as e:
             yield ans + "\n**ERROR**: " + str(e)
 
-        yield total_token_count_from_response(response)
+        if response and hasattr(response, "usage_metadata") and hasattr(response.usage_metadata, "total_token_count"):
+            yield response.usage_metadata.total_token_count
+        else:
+            yield 0
 
 
 class NvidiaCV(Base):
@@ -614,7 +619,7 @@ class NvidiaCV(Base):
         response = response.json()
         return (
             response["choices"][0]["message"]["content"].strip(),
-            total_token_count_from_response(response),
+            response["usage"]["total_tokens"],
         )
 
     def _request(self, msg, gen_conf={}):
@@ -637,7 +642,7 @@ class NvidiaCV(Base):
         response = self._request(vision_prompt)
         return (
             response["choices"][0]["message"]["content"].strip(),
-            total_token_count_from_response(response)
+            response["usage"]["total_tokens"],
         )
 
     def chat(self, system, history, gen_conf, images=[], **kwargs):
@@ -645,7 +650,7 @@ class NvidiaCV(Base):
             response = self._request(self._form_history(system, history, images), gen_conf)
             return (
                 response["choices"][0]["message"]["content"].strip(),
-                total_token_count_from_response(response)
+                response["usage"]["total_tokens"],
             )
         except Exception as e:
             return "**ERROR**: " + str(e), 0
@@ -656,7 +661,7 @@ class NvidiaCV(Base):
             response = self._request(self._form_history(system, history, images), gen_conf)
             cnt = response["choices"][0]["message"]["content"]
             if "usage" in response and "total_tokens" in response["usage"]:
-                total_tokens +=  total_token_count_from_response(response)
+                total_tokens += response["usage"]["total_tokens"]
             for resp in cnt:
                 yield resp
         except Exception as e:

rag/llm/embedding_model.py

@@ -33,7 +33,7 @@ from zhipuai import ZhipuAI
 from api import settings
 from api.utils.file_utils import get_home_cache_dir
 from api.utils.log_utils import log_exception
-from rag.utils import num_tokens_from_string, truncate
+from rag.utils import num_tokens_from_string, truncate, total_token_count_from_response
 
 
 class Base(ABC):
@@ -52,15 +52,7 @@ class Base(ABC):
         raise NotImplementedError("Please implement encode method!")
 
     def total_token_count(self, resp):
-        try:
-            return resp.usage.total_tokens
-        except Exception:
-            pass
-        try:
-            return resp["usage"]["total_tokens"]
-        except Exception:
-            pass
-        return 0
+        return total_token_count_from_response(resp)
 
 
 class DefaultEmbedding(Base):
@@ -944,6 +936,7 @@ class GiteeEmbed(SILICONFLOWEmbed):
             base_url = "https://ai.gitee.com/v1/embeddings"
         super().__init__(key, model_name, base_url)
 
+
 class DeepInfraEmbed(OpenAIEmbed):
     _FACTORY_NAME = "DeepInfra"
 
@@ -969,11 +962,3 @@ class CometAPIEmbed(OpenAIEmbed):
         if not base_url:
             base_url = "https://api.cometapi.com/v1"
         super().__init__(key, model_name, base_url)
-
-class DeerAPIEmbed(OpenAIEmbed):
-    _FACTORY_NAME = "DeerAPI"
-
-    def __init__(self, key, model_name, base_url="https://api.deerapi.com/v1"):
-        if not base_url:
-            base_url = "https://api.deerapi.com/v1"
-        super().__init__(key, model_name, base_url)

rag/llm/sequence2txt_model.py

@@ -244,12 +244,3 @@ class CometAPISeq2txt(Base):
             base_url = "https://api.cometapi.com/v1"
         self.client = OpenAI(api_key=key, base_url=base_url)
         self.model_name = model_name
-        
-class DeerAPISeq2txt(Base):
-    _FACTORY_NAME = "DeerAPI"
-
-    def __init__(self, key, model_name="whisper-1", base_url="https://api.deerapi.com/v1", **kwargs):
-        if not base_url:
-            base_url = "https://api.deerapi.com/v1"
-        self.client = OpenAI(api_key=key, base_url=base_url)
-        self.model_name = model_name

rag/llm/tts_model.py

@@ -402,11 +402,3 @@ class CometAPITTS(OpenAITTS):
         if not base_url:
             base_url = "https://api.cometapi.com/v1"
         super().__init__(key, model_name, base_url, **kwargs)
-        
-class DeerAPITTS(OpenAITTS):
-    _FACTORY_NAME = "DeerAPI"
-
-    def __init__(self, key, model_name, base_url="https://api.deerapi.com/v1", **kwargs):
-        if not base_url:
-            base_url = "https://api.deerapi.com/v1"
-        super().__init__(key, model_name, base_url, **kwargs)

rag/prompts/assign_toc_levels.md

@@ -1,53 +0,0 @@
-You are given a JSON array of TOC items. Each item has at least {"title": string} and may include an existing structure.
-
-Task
-- For each item, assign a depth label using Arabic numerals only: top-level = 1, second-level = 2, third-level = 3, etc.
-- Multiple items may share the same depth (e.g., many 1s, many 2s).
-- Do not use dotted numbering (no 1.1/1.2). Use a single digit string per item indicating its depth only.
-- Preserve the original item order exactly. Do not insert, delete, or reorder.
-- Decide levels yourself to keep a coherent hierarchy. Keep peers at the same depth.
-
-Output
-- Return a valid JSON array only (no extra text).
-- Each element must be {"structure": "1|2|3", "title": <original title string>}.
-- title must be the original title string.
-
-Examples
-
-Example A (chapters with sections)
-Input:
-["Chapter 1 Methods", "Section 1 Definition", "Section 2 Process", "Chapter 2 Experiment"]
-
-Output:
-[
-  {"structure":"1","title":"Chapter 1 Methods"},
-  {"structure":"2","title":"Section 1 Definition"},
-  {"structure":"2","title":"Section 2 Process"},
-  {"structure":"1","title":"Chapter 2 Experiment"}
-]
-
-Example B (parts with chapters)
-Input:
-["Part I Theory", "Chapter 1 Basics", "Chapter 2 Methods", "Part II Applications", "Chapter 3 Case Studies"]
-
-Output:
-[
-  {"structure":"1","title":"Part I Theory"},
-  {"structure":"2","title":"Chapter 1 Basics"},
-  {"structure":"2","title":"Chapter 2 Methods"},
-  {"structure":"1","title":"Part II Applications"},
-  {"structure":"2","title":"Chapter 3 Case Studies"}
-]
-
-Example C (plain headings)
-Input:
-["Introduction", "Background and Motivation", "Related Work", "Methodology", "Evaluation"]
-
-Output:
-[
-  {"structure":"1","title":"Introduction"},
-  {"structure":"2","title":"Background and Motivation"},
-  {"structure":"2","title":"Related Work"},
-  {"structure":"1","title":"Methodology"},
-  {"structure":"1","title":"Evaluation"}
-]

rag/prompts/generator.py

@@ -29,7 +29,7 @@ from rag.utils import encoder, num_tokens_from_string
 
 STOP_TOKEN="<|STOP|>"
 COMPLETE_TASK="complete_task"
-INPUT_UTILIZATION = 0.5
+
 
 def get_value(d, k1, k2):
     return d.get(k1, d.get(k2))
@@ -439,9 +439,9 @@ def gen_meta_filter(chat_mdl, meta_data:dict, query: str) -> list:
     return []
 
 
-def gen_json(system_prompt:str, user_prompt:str, chat_mdl, gen_conf = None):
+def gen_json(system_prompt:str, user_prompt:str, chat_mdl):
     _, msg = message_fit_in(form_message(system_prompt, user_prompt), chat_mdl.max_length)
-    ans = chat_mdl.chat(msg[0]["content"], msg[1:],gen_conf=gen_conf)
+    ans = chat_mdl.chat(msg[0]["content"], msg[1:])
     ans = re.sub(r"(^.*</think>|```json\n|```\n*$)", "", ans, flags=re.DOTALL)
     try:
         return json_repair.loads(ans)
@@ -649,85 +649,4 @@ def toc_transformer(toc_pages, chat_mdl):
     return last_complete
 
 
-TOC_LEVELS = load_prompt("assign_toc_levels")
-def assign_toc_levels(toc_secs, chat_mdl, gen_conf = {"temperature": 0.2}):
-    print("\nBegin TOC level assignment...\n")
 
-    ans = gen_json(
-        PROMPT_JINJA_ENV.from_string(TOC_LEVELS).render(),
-        str(toc_secs),
-        chat_mdl,
-        gen_conf
-    )
-    
-    return ans
-
-
-TOC_FROM_TEXT_SYSTEM = load_prompt("toc_from_text_system")
-TOC_FROM_TEXT_USER = load_prompt("toc_from_text_user")
-# Generate TOC from text chunks with text llms
-def gen_toc_from_text(text, chat_mdl):
-    ans = gen_json(
-        PROMPT_JINJA_ENV.from_string(TOC_FROM_TEXT_SYSTEM).render(),
-        PROMPT_JINJA_ENV.from_string(TOC_FROM_TEXT_USER).render(text=text),
-        chat_mdl,
-        gen_conf={"temperature": 0.0, "top_p": 0.9, "enable_thinking": False, }
-    )
-    return ans
-
-
-def split_chunks(chunks, max_length: int):
-    """
-    Pack chunks into batches according to max_length, returning [{"id": idx, "text": chunk_text}, ...].
-    Do not split a single chunk, even if it exceeds max_length.
-    """
-
-    result = []
-    batch, batch_tokens = [], 0
-
-    for idx, chunk in enumerate(chunks):
-        t = num_tokens_from_string(chunk)
-        if batch_tokens + t > max_length:
-            result.append(batch)
-            batch, batch_tokens = [], 0
-        batch.append({"id": idx, "text": chunk})    
-        batch_tokens += t
-    if batch:
-        result.append(batch)
-    return result
-
-
-def run_toc_from_text(chunks, chat_mdl):
-    input_budget = int(chat_mdl.max_length * INPUT_UTILIZATION) - num_tokens_from_string(
-        TOC_FROM_TEXT_USER + TOC_FROM_TEXT_SYSTEM
-    )
-
-    input_budget =  2000 if input_budget > 2000 else input_budget
-    chunk_sections = split_chunks(chunks, input_budget)
-    res = []
-
-    for chunk in chunk_sections:
-        ans = gen_toc_from_text(chunk, chat_mdl)
-        res.extend(ans)
-        
-    # Filter out entries with title == -1
-    filtered = [x for x in res if x.get("title") and x.get("title") != "-1"]
-
-    print("\n\nFiltered TOC sections:\n", filtered)
-
-    # Generate initial structure (structure/title)
-    raw_structure = [{"structure": "0", "title": x.get("title", "")} for x in filtered]
-
-    # Assign hierarchy levels using LLM
-    toc_with_levels = assign_toc_levels(raw_structure, chat_mdl, {"temperature": 0.0, "top_p": 0.9, "enable_thinking": False})
-
-    # Merge structure and content (by index)
-    merged = []
-    for _ , (toc_item, src_item) in enumerate(zip(toc_with_levels, filtered)):
-        merged.append({
-            "structure": toc_item.get("structure", "0"),
-            "title": toc_item.get("title", ""),
-            "content": src_item.get("content", ""),
-        })
-
-    return merged

rag/prompts/toc_from_text_system.md

@@ -1,113 +0,0 @@
-You are a robust Table-of-Contents (TOC) extractor.
-
-GOAL
-Given a dictionary of chunks {chunk_id: chunk_text}, extract TOC-like headings and return a strict JSON array of objects:
-[
-  {"title": , "content": ""},
-  ...
-]
-
-FIELDS
-- "title": the heading text (clean, no page numbers or leader dots).
-  - If any part of a chunk has no valid heading, output that part as {"title":"-1", ...}.
-- "content": the chunk_id (string).
-  - One chunk can yield multiple JSON objects in order (unmatched text + one or more headings).
-
-RULES
-1) Preserve input chunk order strictly.
-2) If a chunk contains multiple headings, expand them in order:
-   - Pre-heading narrative → {"title":"-1","content":chunk_id}
-   - Then each heading → {"title":"...","content":chunk_id}
-3) Do not merge outputs across chunks; each object refers to exactly one chunk_id.
-4) "title" must be non-empty (or exactly "-1"). "content" must be a string (chunk_id).
-5) When ambiguous, prefer "-1" unless the text strongly looks like a heading.
-
-HEADING DETECTION (cues, not hard rules)
-- Appears near line start, short isolated phrase, often followed by content.
-- May contain separators: — —— - : ： · •
-- Numbering styles:
-  • 第[一二三四五六七八九十百]+(篇|章|节|条)
-  • [(（]?[一二三四五六七八九十]+[)）]?
-  • [(（]?[①②③④⑤⑥⑦⑧⑨⑩][)）]?
-  • ^\d+(\.\d+)*[)．.]?\s*
-  • ^[IVXLCDM]+[).]
-  • ^[A-Z][).]
-- Canonical section cues (general only):
-  Common heading indicators include words such as:
-  "Overview", "Introduction", "Background", "Purpose", "Scope", "Definition",
-  "Method", "Procedure", "Result", "Discussion", "Summary", "Conclusion",
-  "Appendix", "Reference", "Annex", "Acknowledgment", "Disclaimer".
-  These are soft cues, not strict requirements.
-- Length restriction:
-  • Chinese heading: ≤25 characters
-  • English heading: ≤80 characters
-- Exclude long narrative sentences, continuous prose, or bullet-style lists → output as "-1".
-
-OUTPUT FORMAT
-- Return ONLY a valid JSON array of {"title","content"} objects.
-- No reasoning or commentary.
-
-EXAMPLES
-
-Example 1 — No heading
-Input:
-{0: "Copyright page · Publication info (ISBN 123-456). All rights reserved."}
-Output:
-[
-  {"title":"-1","content":"0"}
-]
-
-Example 2 — One heading
-Input:
-{1: "Chapter 1: General Provisions This chapter defines the overall rules…"}
-Output:
-[
-  {"title":"Chapter 1: General Provisions","content":"1"}
-]
-
-Example 3 — Narrative + heading
-Input:
-{2: "This paragraph introduces the background and goals. Section 2: Definitions Key terms are explained…"}
-Output:
-[
-  {"title":"-1","content":"2"},
-  {"title":"Section 2: Definitions","content":"2"}
-]
-
-Example 4 — Multiple headings in one chunk
-Input:
-{3: "Declarations and Commitments (I) Party B commits… (II) Party C commits… Appendix A Data Specification"}
-Output:
-[
-  {"title":"Declarations and Commitments (I)","content":"3"},
-  {"title":"(II)","content":"3"},
-  {"title":"Appendix A","content":"3"}
-]
-
-Example 5 — Numbering styles
-Input:
-{4: "1. Scope: Defines boundaries. 2) Definitions: Terms used. III) Methods Overview."}
-Output:
-[
-  {"title":"1. Scope","content":"4"},
-  {"title":"2) Definitions","content":"4"},
-  {"title":"III) Methods","content":"4"}
-]
-
-Example 6 — Long list (NOT headings)
-Input:
-{5: "Item list: apples, bananas, strawberries, blueberries, mangos, peaches"}
-Output:
-[
-  {"title":"-1","content":"5"}
-]
-
-Example 7 — Mixed Chinese/English
-Input:
-{6: "（出版信息略）This standard follows industry practices. Chapter 1: Overview 摘要… 第2节：术语与缩略语"}
-Output:
-[
-  {"title":"-1","content":"6"},
-  {"title":"Chapter 1: Overview","content":"6"},
-  {"title":"第2节：术语与缩略语","content":"6"}
-]

rag/prompts/toc_from_text_user.md

@@ -1,8 +0,0 @@
-OUTPUT FORMAT
-- Return ONLY the JSON array.
-- Use double quotes.
-- No extra commentary.
-- Keep language of "title" the same as the input.
-
-INPUT
-{{text}}

rag/prompts/toc_relevance_system.md

@@ -1,118 +0,0 @@
-# System Prompt: TOC Relevance Evaluation
-
-You are an expert logical reasoning assistant specializing in hierarchical Table of Contents (TOC) relevance evaluation.
-
-## GOAL
-You will receive:
-1. A JSON list of TOC items, each with fields:
-   ```json
-   {
-     "level": <integer>,   // e.g., 1, 2, 3
-     "title": <string>     // section title
-   }
-   ```
-2. A user query (natural language question).
-
-You must assign a **relevance score** (integer) to every TOC entry, based on how related its `title` is to the `query`.
-
----
-
-## RULES
-
-### Scoring System
-- 5 → highly relevant (directly answers or matches the query intent)
-- 3 → somewhat related (same topic or partially overlaps)
-- 1 → weakly related (vague or tangential)
-- 0 → no clear relation
-- -1 → explicitly irrelevant or contradictory
-
-### Hierarchy Traversal
-- The TOC is hierarchical: smaller `level` = higher layer (e.g., level 1 is top-level, level 2 is a subsection).
-- You must traverse in **hierarchical order** — interpret the structure based on levels (1 > 2 > 3).
-- If a high-level item (level 1) is strongly related (score 5), its child items (level 2, 3) are likely relevant too.
-- If a high-level item is unrelated (-1 or 0), its deeper children are usually less relevant unless the titles clearly match the query.
-- Lower (deeper) levels provide more specific content; prefer assigning higher scores if they directly match the query.
-
-### Output Format
-Return a **JSON array**, preserving the input order but adding a new key `"score"`:
-
-```json
-[
-  {"level": 1, "title": "Introduction", "score": 0},
-  {"level": 2, "title": "Definition of Sustainability", "score": 5}
-]
-```
-
-### Constraints
-- Output **only the JSON array** — no explanations or reasoning text.
-
-### EXAMPLES
-
-#### Example 1
-Input TOC:
-[
-  {"level": 1, "title": "Machine Learning Overview"},
-  {"level": 2, "title": "Supervised Learning"},
-  {"level": 2, "title": "Unsupervised Learning"},
-  {"level": 3, "title": "Applications of Deep Learning"}
-]
-
-Query:
-"How is deep learning used in image classification?"
-
-Output:
-[
-  {"level": 1, "title": "Machine Learning Overview", "score": 3},
-  {"level": 2, "title": "Supervised Learning", "score": 3},
-  {"level": 2, "title": "Unsupervised Learning", "score": 0},
-  {"level": 3, "title": "Applications of Deep Learning", "score": 5}
-]
-
----
-
-#### Example 2
-Input TOC:
-[
-  {"level": 1, "title": "Marketing Basics"},
-  {"level": 2, "title": "Consumer Behavior"},
-  {"level": 2, "title": "Digital Marketing"},
-  {"level": 3, "title": "Social Media Campaigns"},
-  {"level": 3, "title": "SEO Optimization"}
-]
-
-Query:
-"What are the best online marketing methods?"
-
-Output:
-[
-  {"level": 1, "title": "Marketing Basics", "score": 3},
-  {"level": 2, "title": "Consumer Behavior", "score": 1},
-  {"level": 2, "title": "Digital Marketing", "score": 5},
-  {"level": 3, "title": "Social Media Campaigns", "score": 5},
-  {"level": 3, "title": "SEO Optimization", "score": 5}
-]
-
----
-
-#### Example 3
-Input TOC:
-[
-  {"level": 1, "title": "Physics Overview"},
-  {"level": 2, "title": "Classical Mechanics"},
-  {"level": 3, "title": "Newton’s Laws"},
-  {"level": 2, "title": "Thermodynamics"},
-  {"level": 3, "title": "Entropy and Heat Transfer"}
-]
-
-Query:
-"What is entropy?"
-
-Output:
-[
-  {"level": 1, "title": "Physics Overview", "score": 3},
-  {"level": 2, "title": "Classical Mechanics", "score": 0},
-  {"level": 3, "title": "Newton’s Laws", "score": -1},
-  {"level": 2, "title": "Thermodynamics", "score": 5},
-  {"level": 3, "title": "Entropy and Heat Transfer", "score": 5}
-]
-

rag/prompts/toc_relevance_user.md

@@ -1,17 +0,0 @@
-# User Prompt: TOC Relevance Evaluation
-
-You will now receive:
-1. A JSON list of TOC items (each with `level` and `title`)
-2. A user query string.
-
-Traverse the TOC hierarchically based on level numbers and assign scores (5,3,1,0,-1) according to the rules in the system prompt.  
-Output **only** the JSON array with the added `"score"` field.
-
----
-
-**Input TOC:**
-{{ toc_json }}
-
-**Query:**
-{{ query }}
-

rag/svr/task_executor.py

@@ -32,7 +32,7 @@ from api.utils.log_utils import init_root_logger, get_project_base_directory
 from graphrag.general.index import run_graphrag_for_kb
 from graphrag.utils import get_llm_cache, set_llm_cache, get_tags_from_cache, set_tags_to_cache
 from rag.flow.pipeline import Pipeline
-from rag.prompts.generator import keyword_extraction, question_proposal, content_tagging
+from rag.prompts import keyword_extraction, question_proposal, content_tagging
 import logging
 import os
 from datetime import datetime
@@ -380,7 +380,7 @@ async def build_chunks(task, progress_callback):
         examples = []
         all_tags = get_tags_from_cache(kb_ids)
         if not all_tags:
-            all_tags = settings.retriever.all_tags_in_portion(tenant_id, kb_ids, S)
+            all_tags = settings.retrievaler.all_tags_in_portion(tenant_id, kb_ids, S)
             set_tags_to_cache(kb_ids, all_tags)
         else:
             all_tags = json.loads(all_tags)
@@ -393,7 +393,7 @@ async def build_chunks(task, progress_callback):
             if task_canceled:
                 progress_callback(-1, msg="Task has been canceled.")
                 return
-            if settings.retriever.tag_content(tenant_id, kb_ids, d, all_tags, topn_tags=topn_tags, S=S) and len(d[TAG_FLD]) > 0:
+            if settings.retrievaler.tag_content(tenant_id, kb_ids, d, all_tags, topn_tags=topn_tags, S=S) and len(d[TAG_FLD]) > 0:
                 examples.append({"content": d["content_with_weight"], TAG_FLD: d[TAG_FLD]})
             else:
                 docs_to_tag.append(d)
@@ -645,7 +645,7 @@ async def run_raptor_for_kb(row, kb_parser_config, chat_mdl, embd_mdl, vector_si
     chunks = []
     vctr_nm = "q_%d_vec"%vector_size
     for doc_id in doc_ids:
-        for d in settings.retriever.chunk_list(doc_id, row["tenant_id"], [str(row["kb_id"])],
+        for d in settings.retrievaler.chunk_list(doc_id, row["tenant_id"], [str(row["kb_id"])],
                                                  fields=["content_with_weight", vctr_nm],
                                                  sort_by_position=True):
             chunks.append((d["content_with_weight"], np.array(d[vctr_nm])))

rag/utils/__init__.py

@@ -95,12 +95,6 @@ def total_token_count_from_response(resp):
         except Exception:
             pass
 
-    if hasattr(resp, "usage_metadata") and hasattr(resp.usage_metadata, "total_tokens"):
-        try:
-            return resp.usage_metadata.total_tokens
-        except Exception:
-            pass
-
     if 'usage' in resp and 'total_tokens' in resp['usage']:
         try:
             return resp["usage"]["total_tokens"]

rag/utils/es_conn.py

@@ -28,7 +28,6 @@ from rag import settings
 from rag.settings import TAG_FLD, PAGERANK_FLD
 from rag.utils import singleton, get_float
 from api.utils.file_utils import get_project_base_directory
-from api.utils.common import convert_bytes
 from rag.utils.doc_store_conn import DocStoreConnection, MatchExpr, OrderByExpr, MatchTextExpr, MatchDenseExpr, \
     FusionExpr
 from rag.nlp import is_english, rag_tokenizer
@@ -580,52 +579,3 @@ class ESConnection(DocStoreConnection):
                 break
         logger.error(f"ESConnection.sql timeout for {ATTEMPT_TIME} times!")
         return None
-
-    def get_cluster_stats(self):
-        """
-        curl -XGET "http://{es_host}/_cluster/stats" -H "kbn-xsrf: reporting" to view raw stats.
-        """
-        raw_stats = self.es.cluster.stats()
-        logger.debug(f"ESConnection.get_cluster_stats: {raw_stats}")
-        try:
-            res = {
-                'cluster_name': raw_stats['cluster_name'],
-                'status': raw_stats['status']
-            }
-            indices_status = raw_stats['indices']
-            res.update({
-                'indices': indices_status['count'],
-                'indices_shards': indices_status['shards']['total']
-            })
-            doc_info = indices_status['docs']
-            res.update({
-                'docs': doc_info['count'],
-                'docs_deleted': doc_info['deleted']
-            })
-            store_info = indices_status['store']
-            res.update({
-                'store_size': convert_bytes(store_info['size_in_bytes']),
-                'total_dataset_size': convert_bytes(store_info['total_data_set_size_in_bytes'])
-            })
-            mappings_info = indices_status['mappings']
-            res.update({
-                'mappings_fields': mappings_info['total_field_count'],
-                'mappings_deduplicated_fields': mappings_info['total_deduplicated_field_count'],
-                'mappings_deduplicated_size': convert_bytes(mappings_info['total_deduplicated_mapping_size_in_bytes'])
-            })
-            node_info = raw_stats['nodes']
-            res.update({
-                'nodes': node_info['count']['total'],
-                'nodes_version': node_info['versions'],
-                'os_mem': convert_bytes(node_info['os']['mem']['total_in_bytes']),
-                'os_mem_used': convert_bytes(node_info['os']['mem']['used_in_bytes']),
-                'os_mem_used_percent': node_info['os']['mem']['used_percent'],
-                'jvm_versions': node_info['jvm']['versions'][0]['vm_version'],
-                'jvm_heap_used': convert_bytes(node_info['jvm']['mem']['heap_used_in_bytes']),
-                'jvm_heap_max': convert_bytes(node_info['jvm']['mem']['heap_max_in_bytes'])
-            })
-            return res
-
-        except Exception as e:
-            logger.exception(f"ESConnection.get_cluster_stats: {e}")
-            return None

rag/utils/infinity_conn.py

@@ -30,7 +30,6 @@ from rag.settings import PAGERANK_FLD, TAG_FLD
 from rag.utils import singleton
 import pandas as pd
 from api.utils.file_utils import get_project_base_directory
-from rag.nlp import is_english
 
 from rag.utils.doc_store_conn import (
     DocStoreConnection,
@@ -41,15 +40,13 @@ from rag.utils.doc_store_conn import (
     OrderByExpr,
 )
 
-logger = logging.getLogger("ragflow.infinity_conn")
-
+logger = logging.getLogger('ragflow.infinity_conn')
 
 def field_keyword(field_name: str):
-    # The "docnm_kwd" field is always a string, not list.
-    if field_name == "source_id" or (field_name.endswith("_kwd") and field_name != "docnm_kwd" and field_name != "knowledge_graph_kwd"):
-        return True
-    return False
-
+        # The "docnm_kwd" field is always a string, not list.
+        if field_name == "source_id" or (field_name.endswith("_kwd") and field_name != "docnm_kwd" and field_name != "knowledge_graph_kwd"):
+            return True
+        return False
 
 def equivalent_condition_to_str(condition: dict, table_instance=None) -> str | None:
     assert "_id" not in condition
@@ -77,7 +74,7 @@ def equivalent_condition_to_str(condition: dict, table_instance=None) -> str | N
                 inCond = list()
                 for item in v:
                     if isinstance(item, str):
-                        item = item.replace("'", "''")
+                        item = item.replace("'","''")
                     inCond.append(f"filter_fulltext('{k}', '{item}')")
                 if inCond:
                     strInCond = " or ".join(inCond)
@@ -89,7 +86,7 @@ def equivalent_condition_to_str(condition: dict, table_instance=None) -> str | N
             inCond = list()
             for item in v:
                 if isinstance(item, str):
-                    item = item.replace("'", "''")
+                    item = item.replace("'","''")
                     inCond.append(f"'{item}'")
                 else:
                     inCond.append(str(item))
@@ -115,13 +112,13 @@ def concat_dataframes(df_list: list[pd.DataFrame], selectFields: list[str]) -> p
     df_list2 = [df for df in df_list if not df.empty]
     if df_list2:
         return pd.concat(df_list2, axis=0).reset_index(drop=True)
-
+    
     schema = []
     for field_name in selectFields:
-        if field_name == "score()":  # Workaround: fix schema is changed to score()
-            schema.append("SCORE")
-        elif field_name == "similarity()":  # Workaround: fix schema is changed to similarity()
-            schema.append("SIMILARITY")
+        if field_name == 'score()': # Workaround: fix schema is changed to score()
+            schema.append('SCORE')
+        elif field_name == 'similarity()': # Workaround: fix schema is changed to similarity()
+            schema.append('SIMILARITY')
         else:
             schema.append(field_name)
     return pd.DataFrame(columns=schema)
@@ -161,7 +158,9 @@ class InfinityConnection(DocStoreConnection):
 
     def _migrate_db(self, inf_conn):
         inf_db = inf_conn.create_database(self.dbName, ConflictType.Ignore)
-        fp_mapping = os.path.join(get_project_base_directory(), "conf", "infinity_mapping.json")
+        fp_mapping = os.path.join(
+            get_project_base_directory(), "conf", "infinity_mapping.json"
+        )
         if not os.path.exists(fp_mapping):
             raise Exception(f"Mapping file not found at {fp_mapping}")
         schema = json.load(open(fp_mapping))
@@ -179,12 +178,16 @@ class InfinityConnection(DocStoreConnection):
                     continue
                 res = inf_table.add_columns({field_name: field_info})
                 assert res.error_code == infinity.ErrorCode.OK
-                logger.info(f"INFINITY added following column to table {table_name}: {field_name} {field_info}")
+                logger.info(
+                    f"INFINITY added following column to table {table_name}: {field_name} {field_info}"
+                )
                 if field_info["type"] != "varchar" or "analyzer" not in field_info:
                     continue
                 inf_table.create_index(
                     f"text_idx_{field_name}",
-                    IndexInfo(field_name, IndexType.FullText, {"ANALYZER": field_info["analyzer"]}),
+                    IndexInfo(
+                        field_name, IndexType.FullText, {"ANALYZER": field_info["analyzer"]}
+                    ),
                     ConflictType.Ignore,
                 )
 
@@ -218,7 +221,9 @@ class InfinityConnection(DocStoreConnection):
         inf_conn = self.connPool.get_conn()
         inf_db = inf_conn.create_database(self.dbName, ConflictType.Ignore)
 
-        fp_mapping = os.path.join(get_project_base_directory(), "conf", "infinity_mapping.json")
+        fp_mapping = os.path.join(
+            get_project_base_directory(), "conf", "infinity_mapping.json"
+        )
         if not os.path.exists(fp_mapping):
             raise Exception(f"Mapping file not found at {fp_mapping}")
         schema = json.load(open(fp_mapping))
@@ -248,11 +253,15 @@ class InfinityConnection(DocStoreConnection):
                 continue
             inf_table.create_index(
                 f"text_idx_{field_name}",
-                IndexInfo(field_name, IndexType.FullText, {"ANALYZER": field_info["analyzer"]}),
+                IndexInfo(
+                    field_name, IndexType.FullText, {"ANALYZER": field_info["analyzer"]}
+                ),
                 ConflictType.Ignore,
             )
         self.connPool.release_conn(inf_conn)
-        logger.info(f"INFINITY created table {table_name}, vector size {vectorSize}")
+        logger.info(
+            f"INFINITY created table {table_name}, vector size {vectorSize}"
+        )
 
     def deleteIdx(self, indexName: str, knowledgebaseId: str):
         table_name = f"{indexName}_{knowledgebaseId}"
@@ -279,21 +288,20 @@ class InfinityConnection(DocStoreConnection):
     """
 
     def search(
-        self,
-        selectFields: list[str],
-        highlightFields: list[str],
-        condition: dict,
-        matchExprs: list[MatchExpr],
-        orderBy: OrderByExpr,
-        offset: int,
-        limit: int,
-        indexNames: str | list[str],
-        knowledgebaseIds: list[str],
-        aggFields: list[str] = [],
-        rank_feature: dict | None = None,
+            self, selectFields: list[str],
+            highlightFields: list[str],
+            condition: dict,
+            matchExprs: list[MatchExpr],
+            orderBy: OrderByExpr,
+            offset: int,
+            limit: int,
+            indexNames: str | list[str],
+            knowledgebaseIds: list[str],
+            aggFields: list[str] = [],
+            rank_feature: dict | None = None
     ) -> tuple[pd.DataFrame, int]:
         """
-        BUG: Infinity returns empty for a highlight field if the query string doesn't use that field.
+        TODO: Infinity doesn't provide highlight
         """
         if isinstance(indexNames, str):
             indexNames = indexNames.split(",")
@@ -430,7 +438,9 @@ class InfinityConnection(DocStoreConnection):
                                 matchExpr.extra_options.copy(),
                             )
                         elif isinstance(matchExpr, FusionExpr):
-                            builder = builder.fusion(matchExpr.method, matchExpr.topn, matchExpr.fusion_params)
+                            builder = builder.fusion(
+                                matchExpr.method, matchExpr.topn, matchExpr.fusion_params
+                            )
                 else:
                     if filter_cond and len(filter_cond) > 0:
                         builder.filter(filter_cond)
@@ -445,13 +455,15 @@ class InfinityConnection(DocStoreConnection):
         self.connPool.release_conn(inf_conn)
         res = concat_dataframes(df_list, output)
         if matchExprs:
-            res["Sum"] = res[score_column] + res[PAGERANK_FLD]
-            res = res.sort_values(by="Sum", ascending=False).reset_index(drop=True).drop(columns=["Sum"])
+            res['Sum'] = res[score_column] + res[PAGERANK_FLD]
+            res = res.sort_values(by='Sum', ascending=False).reset_index(drop=True).drop(columns=['Sum'])
             res = res.head(limit)
         logger.debug(f"INFINITY search final result: {str(res)}")
         return res, total_hits_count
 
-    def get(self, chunkId: str, indexName: str, knowledgebaseIds: list[str]) -> dict | None:
+    def get(
+            self, chunkId: str, indexName: str, knowledgebaseIds: list[str]
+    ) -> dict | None:
         inf_conn = self.connPool.get_conn()
         db_instance = inf_conn.get_database(self.dbName)
         df_list = list()
@@ -464,7 +476,8 @@ class InfinityConnection(DocStoreConnection):
             try:
                 table_instance = db_instance.get_table(table_name)
             except Exception:
-                logger.warning(f"Table not found: {table_name}, this knowledge base isn't created in Infinity. Maybe it is created in other document engine.")
+                logger.warning(
+                    f"Table not found: {table_name}, this knowledge base isn't created in Infinity. Maybe it is created in other document engine.")
                 continue
             kb_res, _ = table_instance.output(["*"]).filter(f"id = '{chunkId}'").to_df()
             logger.debug(f"INFINITY get table: {str(table_list)}, result: {str(kb_res)}")
@@ -474,7 +487,9 @@ class InfinityConnection(DocStoreConnection):
         res_fields = self.getFields(res, res.columns.tolist())
         return res_fields.get(chunkId, None)
 
-    def insert(self, documents: list[dict], indexName: str, knowledgebaseId: str = None) -> list[str]:
+    def insert(
+            self, documents: list[dict], indexName: str, knowledgebaseId: str = None
+    ) -> list[str]:
         inf_conn = self.connPool.get_conn()
         db_instance = inf_conn.get_database(self.dbName)
         table_name = f"{indexName}_{knowledgebaseId}"
@@ -517,7 +532,7 @@ class InfinityConnection(DocStoreConnection):
                         d[k] = v
                 elif re.search(r"_feas$", k):
                     d[k] = json.dumps(v)
-                elif k == "kb_id":
+                elif k == 'kb_id':
                     if isinstance(d[k], list):
                         d[k] = d[k][0]  # since d[k] is a list, but we need a str
                 elif k == "position_int":
@@ -546,16 +561,18 @@ class InfinityConnection(DocStoreConnection):
         logger.debug(f"INFINITY inserted into {table_name} {str_ids}.")
         return []
 
-    def update(self, condition: dict, newValue: dict, indexName: str, knowledgebaseId: str) -> bool:
+    def update(
+            self, condition: dict, newValue: dict, indexName: str, knowledgebaseId: str
+    ) -> bool:
         # if 'position_int' in newValue:
         #     logger.info(f"update position_int: {newValue['position_int']}")
         inf_conn = self.connPool.get_conn()
         db_instance = inf_conn.get_database(self.dbName)
         table_name = f"{indexName}_{knowledgebaseId}"
         table_instance = db_instance.get_table(table_name)
-        # if "exists" in condition:
+        #if "exists" in condition:
         #    del condition["exists"]
-
+        
         clmns = {}
         if table_instance:
             for n, ty, de, _ in table_instance.show_columns().rows():
@@ -570,7 +587,7 @@ class InfinityConnection(DocStoreConnection):
                     newValue[k] = v
             elif re.search(r"_feas$", k):
                 newValue[k] = json.dumps(v)
-            elif k == "kb_id":
+            elif k == 'kb_id':
                 if isinstance(newValue[k], list):
                     newValue[k] = newValue[k][0]  # since d[k] is a list, but we need a str
             elif k == "position_int":
@@ -594,11 +611,11 @@ class InfinityConnection(DocStoreConnection):
                     del newValue[k]
             else:
                 newValue[k] = v
-
-        remove_opt = {}  # "[k,new_value]": [id_to_update, ...]
+                
+        remove_opt = {}     # "[k,new_value]": [id_to_update, ...]
         if removeValue:
             col_to_remove = list(removeValue.keys())
-            row_to_opt = table_instance.output(col_to_remove + ["id"]).filter(filter).to_df()
+            row_to_opt = table_instance.output(col_to_remove + ['id']).filter(filter).to_df()
             logger.debug(f"INFINITY search table {str(table_name)}, filter {filter}, result: {str(row_to_opt[0])}")
             row_to_opt = self.getFields(row_to_opt, col_to_remove)
             for id, old_v in row_to_opt.items():
@@ -615,8 +632,8 @@ class InfinityConnection(DocStoreConnection):
         logger.debug(f"INFINITY update table {table_name}, filter {filter}, newValue {newValue}.")
         for update_kv, ids in remove_opt.items():
             k, v = json.loads(update_kv)
-            table_instance.update(filter + " AND id in ({0})".format(",".join([f"'{id}'" for id in ids])), {k: "###".join(v)})
-
+            table_instance.update(filter + " AND id in ({0})".format(",".join([f"'{id}'" for id in ids])), {k:"###".join(v)})
+                
         table_instance.update(filter, newValue)
         self.connPool.release_conn(inf_conn)
         return True
@@ -628,7 +645,9 @@ class InfinityConnection(DocStoreConnection):
         try:
             table_instance = db_instance.get_table(table_name)
         except Exception:
-            logger.warning(f"Skipped deleting from table {table_name} since the table doesn't exist.")
+            logger.warning(
+                f"Skipped deleting from table {table_name} since the table doesn't exist."
+            )
             return 0
         filter = equivalent_condition_to_str(condition, table_instance)
         logger.debug(f"INFINITY delete table {table_name}, filter {filter}.")
@@ -656,39 +675,37 @@ class InfinityConnection(DocStoreConnection):
         if not fields:
             return {}
         fieldsAll = fields.copy()
-        fieldsAll.append("id")
+        fieldsAll.append('id')
         column_map = {col.lower(): col for col in res.columns}
-        matched_columns = {column_map[col.lower()]: col for col in set(fieldsAll) if col.lower() in column_map}
+        matched_columns = {column_map[col.lower()]:col for col in set(fieldsAll) if col.lower() in column_map}
         none_columns = [col for col in set(fieldsAll) if col.lower() not in column_map]
 
         res2 = res[matched_columns.keys()]
         res2 = res2.rename(columns=matched_columns)
-        res2.drop_duplicates(subset=["id"], inplace=True)
+        res2.drop_duplicates(subset=['id'], inplace=True)
 
         for column in res2.columns:
             k = column.lower()
             if field_keyword(k):
-                res2[column] = res2[column].apply(lambda v: [kwd for kwd in v.split("###") if kwd])
+                res2[column] = res2[column].apply(lambda v:[kwd for kwd in v.split("###") if kwd])
             elif re.search(r"_feas$", k):
                 res2[column] = res2[column].apply(lambda v: json.loads(v) if v else {})
             elif k == "position_int":
-
                 def to_position_int(v):
                     if v:
-                        arr = [int(hex_val, 16) for hex_val in v.split("_")]
-                        v = [arr[i : i + 5] for i in range(0, len(arr), 5)]
+                        arr = [int(hex_val, 16) for hex_val in v.split('_')]
+                        v = [arr[i:i + 5] for i in range(0, len(arr), 5)]
                     else:
                         v = []
                     return v
-
                 res2[column] = res2[column].apply(to_position_int)
             elif k in ["page_num_int", "top_int"]:
-                res2[column] = res2[column].apply(lambda v: [int(hex_val, 16) for hex_val in v.split("_")] if v else [])
+                res2[column] = res2[column].apply(lambda v:[int(hex_val, 16) for hex_val in v.split('_')] if v else [])
             else:
                 pass
         for column in none_columns:
             res2[column] = None
-
+        
         return res2.set_index("id").to_dict(orient="index")
 
     def getHighlight(self, res: tuple[pd.DataFrame, int] | pd.DataFrame, keywords: list[str], fieldnm: str):
@@ -702,35 +719,23 @@ class InfinityConnection(DocStoreConnection):
         for i in range(num_rows):
             id = column_id[i]
             txt = res[fieldnm][i]
-            if re.search(r"<em>[^<>]+</em>", txt, flags=re.IGNORECASE | re.MULTILINE):
-                ans[id] = txt
-                continue
             txt = re.sub(r"[\r\n]", " ", txt, flags=re.IGNORECASE | re.MULTILINE)
             txts = []
             for t in re.split(r"[.?!;\n]", txt):
-                if is_english([t]):
-                    for w in keywords:
-                        t = re.sub(
-                            r"(^|[ .?/'\"\(\)!,:;-])(%s)([ .?/'\"\(\)!,:;-])" % re.escape(w),
-                            r"\1<em>\2</em>\3",
-                            t,
-                            flags=re.IGNORECASE | re.MULTILINE,
-                        )
-                else:
-                    for w in sorted(keywords, key=len, reverse=True):
-                        t = re.sub(
-                            re.escape(w),
-                            f"<em>{w}</em>",
-                            t,
-                            flags=re.IGNORECASE | re.MULTILINE,
-                        )
-                if not re.search(r"<em>[^<>]+</em>", t, flags=re.IGNORECASE | re.MULTILINE):
+                for w in keywords:
+                    t = re.sub(
+                        r"(^|[ .?/'\"\(\)!,:;-])(%s)([ .?/'\"\(\)!,:;-])"
+                        % re.escape(w),
+                        r"\1<em>\2</em>\3",
+                        t,
+                        flags=re.IGNORECASE | re.MULTILINE,
+                    )
+                if not re.search(
+                        r"<em>[^<>]+</em>", t, flags=re.IGNORECASE | re.MULTILINE
+                ):
                     continue
                 txts.append(t)
-            if txts:
-                ans[id] = "...".join(txts)
-            else:
-                ans[id] = txt
+            ans[id] = "...".join(txts)
         return ans
 
     def getAggregation(self, res: tuple[pd.DataFrame, int] | pd.DataFrame, fieldnm: str):

rag/utils/redis_conn.py

@@ -91,20 +91,6 @@ class RedisDB:
         if self.REDIS.get(a) == b:
             return True
 
-    def info(self):
-        info = self.REDIS.info()
-        return {
-            'redis_version': info["redis_version"],
-            'server_mode': info["server_mode"],
-            'used_memory': info["used_memory_human"],
-            'total_system_memory': info["total_system_memory_human"],
-            'mem_fragmentation_ratio': info["mem_fragmentation_ratio"],
-            'connected_clients': info["connected_clients"],
-            'blocked_clients': info["blocked_clients"],
-            'instantaneous_ops_per_sec': info["instantaneous_ops_per_sec"],
-            'total_commands_processed': info["total_commands_processed"]
-        }
-
     def is_alive(self):
         return self.REDIS is not None
 

web/.storybook/main.ts

@@ -3,7 +3,6 @@ import path from 'path';
 
 const config: StorybookConfig = {
   stories: ['../src/**/*.mdx', '../src/**/*.stories.@(js|jsx|mjs|ts|tsx)'],
-  staticDirs: ['../public'],
   addons: [
     '@storybook/addon-webpack5-compiler-swc',
     '@storybook/addon-docs',

web/.storybook/preview.ts

@@ -1,7 +1,6 @@
 import '@/locales/config';
 import type { Preview } from '@storybook/react-webpack5';
 import { createElement } from 'react';
-import '../public/iconfont.js';
 import { TooltipProvider } from '../src/components/ui/tooltip';
 
 import '../tailwind.css';

web/public/iconfont.js

@@ -59,10 +59,6 @@
   `<symbol id="icon-GitHub" viewBox="0 0 1024 1024">
         <path d="M512 42.666667C252.714667 42.666667 42.666667 252.714667 42.666667 512c0 207.658667 134.357333 383.104 320.896 445.269333 23.466667 4.096 32.256-9.941333 32.256-22.272 0-11.178667-0.554667-48.128-0.554667-87.424-117.930667 21.717333-148.437333-28.757333-157.824-55.125333-5.290667-13.525333-28.16-55.168-48.085333-66.304-16.426667-8.832-39.936-30.506667-0.597334-31.104 36.949333-0.597333 63.36 34.005333 72.149334 48.128 42.24 70.954667 109.696 51.029333 136.704 38.698667 4.096-30.506667 16.426667-51.029333 29.909333-62.762667-104.448-11.733333-213.546667-52.224-213.546667-231.765333 0-51.029333 18.176-93.269333 48.128-126.122667-4.736-11.733333-21.162667-59.818667 4.693334-124.373333 0 0 39.296-12.288 129.024 48.128a434.901333 434.901333 0 0 1 117.333333-15.829334c39.936 0 79.829333 5.248 117.333333 15.829334 89.770667-61.013333 129.066667-48.128 129.066667-48.128 25.813333 64.554667 9.429333 112.64 4.736 124.373333 29.909333 32.853333 48.085333 74.538667 48.085333 126.122667 0 180.138667-109.696 220.032-214.144 231.765333 17.024 14.677333 31.701333 42.837333 31.701334 86.826667 0 62.762667-0.597333 113.237333-0.597334 129.066666 0 12.330667 8.789333 26.965333 32.256 22.272C846.976 895.104 981.333333 719.104 981.333333 512c0-259.285333-210.005333-469.333333-469.333333-469.333333z"></path>
    </symbol>` +
-  `<symbol id="icon-more" viewBox="0 0 1024 1024">
-    <path d="M0 0h1024v1024H0z"  opacity=".01"></path>
-    <path d="M867.072 141.184H156.032a32 32 0 0 0 0 64h711.04a32 32 0 0 0 0-64z m0.832 226.368H403.2a32 32 0 0 0 0 64h464.704a32 32 0 0 0 0-64zM403.2 573.888h464.704a32 32 0 0 1 0 64H403.2a32 32 0 0 1 0-64z m464.704 226.368H156.864a32 32 0 0 0 0 64h711.04a32 32 0 0 0 0-64zM137.472 367.552v270.336l174.528-122.24-174.528-148.096z" ></path>
-</symbol>` +
   '</svg>'),
   ((h) => {
     var a = (l = (l = document.getElementsByTagName('script'))[

web/src/assets/svg/llm/deerapi.svg

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Generated by Pixelmator Pro 3.5.5 -->
-<svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
-    <path id="path1" fill="#447eec" stroke="none" d="M 4.043384 3 C 4.218303 3.131369 4.218303 3.131369 4.245634 3.344843 C 4.235589 3.689968 4.120566 4.019037 4.020153 4.354485 C 3.648397 5.639093 3.772068 7.16324 4.721537 8.292472 C 4.834668 8.405636 4.834668 8.405636 5.092898 8.451808 C 5.099438 8.407101 5.105977 8.362394 5.112713 8.316334 C 5.266371 7.465536 5.643018 6.500875 6.492251 5.890116 C 6.722516 5.834181 6.722516 5.834181 6.929549 5.82443 C 7.088069 5.89422 7.088069 5.89422 7.191927 6.021484 C 7.238566 6.266672 7.182186 6.450504 7.071671 6.678328 C 6.587966 7.729729 6.728653 8.892773 7.366847 9.896866 C 7.683047 10.36898 8.067978 10.728178 8.59128 11.079185 C 8.649906 11.121605 8.70853 11.164025 8.768932 11.207732 C 9.66402 11.807772 10.937032 12.285625 12.156346 12.257288 C 13.377048 12.176717 14.726612 11.651421 15.533382 10.972448 C 15.588738 10.922069 15.644094 10.871689 15.701127 10.819783 C 15.850423 10.685079 15.850423 10.685079 16.08547 10.524973 C 16.383444 10.276995 16.547771 10.004078 16.725018 9.699812 C 16.759743 9.646976 16.794468 9.594141 16.830244 9.539706 C 17.098297 9.063352 17.108864 8.602456 17.113121 8.090544 C 17.114 8.021955 17.114882 7.953365 17.115789 7.882698 C 17.109453 7.451891 17.02879 7.068714 16.870663 6.654499 C 16.778524 6.380194 16.726711 6.084627 16.899937 5.82443 C 17.474003 5.873095 17.771864 6.159414 18.124371 6.481276 C 18.275856 6.67165 18.378973 6.862965 18.474209 7.072435 C 18.510059 7.143305 18.510059 7.143305 18.546637 7.215607 C 18.731159 7.597385 18.826818 7.985786 18.911507 8.386124 C 19.354362 8.235567 19.496134 8.016235 19.671312 7.692331 C 19.697708 7.645218 19.724104 7.598103 19.751299 7.549561 C 19.94206 7.198732 20.071728 6.857859 20.135941 6.481276 C 20.153978 6.410828 20.172018 6.340384 20.190603 6.267801 C 20.311863 5.357117 20.176548 4.488987 19.832565 3.61451 C 19.773235 3.416943 19.797062 3.258942 19.873562 3.065685 C 20.343493 3.074291 20.545094 3.203472 20.855154 3.445213 C 21.137327 3.710249 21.304079 4.011683 21.480631 4.321899 C 21.518852 4.385996 21.557068 4.45009 21.596447 4.51613 C 21.93461 5.092474 21.990692 5.56271 21.994455 6.1898 C 21.997162 6.299852 21.997162 6.299852 21.999922 6.412127 C 22.004002 7.072449 21.84753 7.669481 21.491564 8.275281 C 21.453457 8.340647 21.415352 8.406013 21.376089 8.473361 C 21.342043 8.531276 21.307995 8.58919 21.272915 8.648861 C 21.233231 8.726082 21.193544 8.803303 21.152658 8.882862 C 20.590042 9.776692 19.648251 10.593391 18.513157 11.107153 C 17.735565 11.48311 16.590176 12.039842 16.28772 12.721296 C 16.708294 12.597325 17.125742 12.472708 17.534019 12.327189 C 18.42695 12.030186 19.416105 11.804957 20.398319 11.933083 C 20.650791 12.052649 20.650791 12.052649 20.835617 12.195821 C 20.889799 12.745177 20.433676 13.369287 19.976738 13.786102 C 19.068974 14.416512 17.830204 14.815967 16.637558 15.020251 C 16.405041 15.058423 16.405041 15.058423 16.20026 15.15162 C 16.092342 15.326459 16.092342 15.326459 16.047207 15.537517 C 16.0268 15.614313 16.006393 15.691111 15.985371 15.770234 C 15.941998 15.985131 15.916471 16.195107 15.899619 16.41194 C 15.85338 16.919502 15.635668 17.317726 15.243672 17.729734 C 14.687399 18.330616 14.487684 19.071302 14.284009 19.776215 C 14.139319 20.134537 13.878259 20.395561 13.527279 20.656797 C 13.476657 20.695154 13.426036 20.733509 13.373882 20.773027 C 13.044217 20.98155 12.615493 20.960649 12.204453 20.981113 C 12.136414 20.985601 12.068374 20.990088 11.998274 20.994713 C 11.372272 21.025297 10.929299 20.924742 10.406066 20.652691 C 9.956832 20.315306 9.780399 19.975595 9.646261 19.51553 C 9.45176 18.889639 9.236951 18.265442 8.766199 17.721523 C 8.338652 17.217646 8.153504 16.619122 7.973939 16.041952 C 7.951638 15.970907 7.951638 15.970907 7.928885 15.898428 C 7.901113 15.807971 7.874589 15.717293 7.84954 15.62639 C 7.766208 15.349351 7.674052 15.189289 7.366847 15.020251 C 7.084245 14.934055 7.084245 14.934055 6.765562 14.868356 C 5.556292 14.586564 4.309259 14.177955 3.62556 13.33837 C 3.34536 12.926235 3.155031 12.517895 3.343707 12.064452 C 3.860229 11.811237 4.491402 11.904243 5.064542 11.990557 C 5.992092 12.14533 6.804541 12.365472 7.629225 12.721296 C 7.549964 12.200433 7.008167 11.934837 6.498059 11.6257 C 6.238734 11.478328 5.973781 11.345892 5.699649 11.21466 C 5.246062 10.99374 4.881122 10.743555 4.511429 10.448256 C 4.305137 10.283882 4.305137 10.283882 4.043384 10.135485 C 3.752214 9.943573 3.574499 9.753546 3.376505 9.506865 C 3.311633 9.428246 3.246722 9.349648 3.181771 9.271068 C 3.150556 9.232649 3.11934 9.194228 3.087179 9.154644 C 2.999125 9.049258 2.904421 8.947053 2.809384 8.845145 C 1.856228 7.713696 1.874341 6.082378 2.206733 4.810427 C 2.455393 4.168301 3.071521 3.532841 3.693546 3.065685 C 3.872223 3.013599 3.872223 3.013599 4.043384 3 Z"/>
-</svg>

web/src/components/collapse.tsx

@@ -3,16 +3,9 @@ import {
   CollapsibleContent,
   CollapsibleTrigger,
 } from '@/components/ui/collapsible';
-import { cn } from '@/lib/utils';
 import { CollapsibleProps } from '@radix-ui/react-collapsible';
-import {
-  PropsWithChildren,
-  ReactNode,
-  useCallback,
-  useEffect,
-  useState,
-} from 'react';
-import { IconFontFill } from './icon-font';
+import { ListCollapse } from 'lucide-react';
+import { PropsWithChildren, ReactNode } from 'react';
 
 type CollapseProps = Omit<CollapsibleProps, 'title'> & {
   title?: ReactNode;
@@ -23,42 +16,22 @@ export function Collapse({
   title,
   children,
   rightContent,
-  open = true,
+  open,
   defaultOpen = false,
   onOpenChange,
   disabled,
 }: CollapseProps) {
-  const [currentOpen, setCurrentOpen] = useState(open);
-
-  useEffect(() => {
-    setCurrentOpen(open);
-  }, [open]);
-
-  const handleOpenChange = useCallback(
-    (open: boolean) => {
-      setCurrentOpen(open);
-      onOpenChange?.(open);
-    },
-    [onOpenChange],
-  );
-
   return (
     <Collapsible
       defaultOpen={defaultOpen}
-      open={currentOpen}
-      onOpenChange={handleOpenChange}
+      open={open}
+      onOpenChange={onOpenChange}
       disabled={disabled}
     >
       <CollapsibleTrigger className="w-full">
         <section className="flex justify-between items-center pb-2">
           <div className="flex items-center gap-1">
-            <IconFontFill
-              name={`more`}
-              className={cn('size-4', {
-                'rotate-90': !currentOpen,
-              })}
-            ></IconFontFill>
-            {title}
+            <ListCollapse className="size-4" /> {title}
           </div>
           <div>{rightContent}</div>
         </section>

web/src/components/embed-dialog/index.tsx

@@ -233,32 +233,36 @@ function EmbedDialog({
               />
             </form>
           </Form>
-          <div className="max-h-[350px] overflow-auto">
+          <div>
             <span>{t('embedCode', { keyPrefix: 'search' })}</span>
-            <div className="max-h-full overflow-y-auto">
-              <HightLightMarkdown>{text}</HightLightMarkdown>
-            </div>
+            <HightLightMarkdown>{text}</HightLightMarkdown>
+              <div className="max-h-[350px] overflow-auto">
+                <span>{t('embedCode', { keyPrefix: 'search' })}</span>
+                <div className="max-h-full overflow-y-auto">
+                  <HightLightMarkdown>{text}</HightLightMarkdown>
+                </div>
+              </div>
+              <div className=" font-medium mt-4 mb-1">
+                {t(isAgent ? 'flow' : 'chat', { keyPrefix: 'header' })}
+                <span className="ml-1 inline-block">ID</span>
+              </div>
+              <div className="bg-bg-card rounded-lg flex justify-between p-2">
+                <span>{token} </span>
+                <CopyToClipboard text={token}></CopyToClipboard>
+              </div>
+              <a
+                className="cursor-pointer text-accent-primary inline-block"
+                href={
+                  isAgent
+                    ? 'https://ragflow.io/docs/dev/http_api_reference#create-session-with-agent'
+                    : 'https://ragflow.io/docs/dev/http_api_reference#create-session-with-chat-assistant'
+                }
+                target="_blank"
+                rel="noreferrer"
+              >
+                {t('howUseId', { keyPrefix: isAgent ? 'flow' : 'chat' })}
+              </a>
           </div>
-          <div className=" font-medium mt-4 mb-1">
-            {t(isAgent ? 'flow' : 'chat', { keyPrefix: 'header' })}
-            <span className="ml-1 inline-block">ID</span>
-          </div>
-          <div className="bg-bg-card rounded-lg flex justify-between p-2">
-            <span>{token} </span>
-            <CopyToClipboard text={token}></CopyToClipboard>
-          </div>
-          <a
-            className="cursor-pointer text-accent-primary inline-block"
-            href={
-              isAgent
-                ? 'https://ragflow.io/docs/dev/http_api_reference#create-session-with-agent'
-                : 'https://ragflow.io/docs/dev/http_api_reference#create-session-with-chat-assistant'
-            }
-            target="_blank"
-            rel="noreferrer"
-          >
-            {t('howUseId', { keyPrefix: isAgent ? 'flow' : 'chat' })}
-          </a>
         </section>
       </DialogContent>
     </Dialog>

web/src/components/file-status-badge.tsx

@@ -26,7 +26,7 @@ const FileStatusBadge: FC<StatusBadgeProps> = ({ status, name }) => {
       case RunningStatus.UNSTART:
         return `bg-[rgba(250,173,20,0.1)] text-state-warning`;
       default:
-        return 'bg-gray-500/10 text-text-secondary';
+        return 'bg-gray-500/10 text-white';
     }
   };
 
@@ -45,7 +45,7 @@ const FileStatusBadge: FC<StatusBadgeProps> = ({ status, name }) => {
       case RunningStatus.UNSTART:
         return `bg-[rgba(250,173,20,1)] text-state-warning`;
       default:
-        return `bg-[rgba(117,120,122,1)] text-text-secondary`;
+        return 'bg-gray-500/10 text-white';
     }
   };
 

web/src/components/home-card.tsx

@@ -41,7 +41,7 @@ export function HomeCard({
         <div className="flex flex-col justify-between gap-1 flex-1 h-full w-[calc(100%-50px)]">
           <section className="flex justify-between w-full">
             <section className="flex gap-1 items-center w-full">
-              <div className="text-base font-bold w-80% text-ellipsis overflow-hidden leading-snug">
+              <div className="text-[20px] font-bold w-80% leading-5 text-ellipsis overflow-hidden">
                 {data.name}
               </div>
               {icon}

web/src/components/list-filter-bar/interface.ts

@@ -1,7 +1,7 @@
 export type FilterType = {
   id: string;
-  label: string | JSX.Element;
-  count?: number;
+  label: string;
+  count: number;
 };
 
 export type FilterCollection = {

web/src/components/llm-setting-items/llm-form-field.tsx

@@ -4,7 +4,7 @@ import { useTranslation } from 'react-i18next';
 import { SelectWithSearch } from '../originui/select-with-search';
 import { RAGFlowFormItem } from '../ragflow-form';
 
-export type LLMFormFieldProps = {
+type LLMFormFieldProps = {
   options?: any[];
   name?: string;
 };

web/src/components/parse-configuration/raptor-form-fields.tsx

@@ -67,6 +67,19 @@ const RaptorFormFields = ({
   const { t } = useTranslate('knowledgeConfiguration');
   const useRaptor = useWatch({ name: UseRaptorField });
 
+  const changeRaptor = useCallback(
+    (isUseRaptor: boolean) => {
+      if (isUseRaptor) {
+        form.setValue(MaxTokenField, 256);
+        form.setValue(ThresholdField, 0.1);
+        form.setValue(MaxCluster, 64);
+        form.setValue(RandomSeedField, 0);
+        form.setValue(Prompt, t('promptText'));
+      }
+    },
+    [form],
+  );
+
   const handleGenerate = useCallback(() => {
     form.setValue(RandomSeedField, random(10000));
   }, [form]);
@@ -77,6 +90,10 @@ const RaptorFormFields = ({
         control={form.control}
         name={UseRaptorField}
         render={({ field }) => {
+          // if (typeof field.value === 'undefined') {
+          //   // default value set
+          //   form.setValue('parser_config.raptor.use_raptor', false);
+          // }
           return (
             <FormItem
               defaultChecked={false}

web/src/components/slider-input-form-field.tsx

@@ -40,7 +40,7 @@ export function SliderInputFormField({
 }: SliderInputFormFieldProps) {
   const form = useFormContext();
 
-  const isHorizontal = useMemo(() => layout !== FormLayout.Vertical, [layout]);
+  const isHorizontal = useMemo(() => layout === FormLayout.Vertical, [layout]);
 
   return (
     <FormField

web/src/components/ui/command.tsx

@@ -26,7 +26,7 @@ Command.displayName = CommandPrimitive.displayName;
 const CommandDialog = ({ children, ...props }: DialogProps) => {
   return (
     <Dialog {...props}>
-      <DialogContent className="overflow-auto p-0 shadow-lg">
+      <DialogContent className="overflow-hidden p-0 shadow-lg">
         <Command className="[&_[cmdk-group-heading]]:px-2 [&_[cmdk-group-heading]]:font-medium [&_[cmdk-group-heading]]:text-muted-foreground [&_[cmdk-group]:not([hidden])_~[cmdk-group]]:pt-0 [&_[cmdk-group]]:px-2 [&_[cmdk-input-wrapper]_svg]:h-5 [&_[cmdk-input-wrapper]_svg]:w-5 [&_[cmdk-input]]:h-12 [&_[cmdk-item]]:px-2 [&_[cmdk-item]]:py-3 [&_[cmdk-item]_svg]:h-5 [&_[cmdk-item]_svg]:w-5">
           {children}
         </Command>
@@ -58,18 +58,9 @@ const CommandList = React.forwardRef<
   React.ElementRef<typeof CommandPrimitive.List>,
   React.ComponentPropsWithoutRef<typeof CommandPrimitive.List>
 >(({ className, ...props }, ref) => (
-  /**
-   * Solve the problem of the scroll wheel not working
-   * onWheel={(e) => e.stopPropagation()}
-    onMouseEnter={(e) => e.currentTarget.focus()}
-    tabIndex={-1}
-   */
   <CommandPrimitive.List
     ref={ref}
     className={cn('max-h-[300px] overflow-y-auto overflow-x-hidden', className)}
-    onWheel={(e) => e.stopPropagation()}
-    onMouseEnter={(e) => e.currentTarget.focus()}
-    tabIndex={-1}
     {...props}
   />
 ));

web/src/components/ui/tooltip.tsx

@@ -20,7 +20,7 @@ const TooltipContent = React.forwardRef<
     ref={ref}
     sideOffset={sideOffset}
     className={cn(
-      'z-50 overflow-auto scrollbar-auto rounded-md border bg-popover px-3 py-1.5 text-sm text-popover-foreground shadow-md animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 max-w-[20vw]',
+      'z-50 overflow-hidden rounded-md border bg-popover px-3 py-1.5 text-sm text-popover-foreground shadow-md animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 max-w-[20vw]',
       className,
     )}
     {...props}

web/src/constants/agent.ts

@@ -52,13 +52,3 @@ export enum AgentCategory {
   AgentCanvas = 'agent_canvas',
   DataflowCanvas = 'dataflow_canvas',
 }
-
-export enum DataflowOperator {
-  Begin = 'File',
-  Note = 'Note',
-  Parser = 'Parser',
-  Tokenizer = 'Tokenizer',
-  Splitter = 'Splitter',
-  HierarchicalMerger = 'HierarchicalMerger',
-  Extractor = 'Extractor',
-}

web/src/constants/llm.ts

@@ -57,7 +57,6 @@ export enum LLMFactory {
   TokenPony = 'TokenPony',
   Meituan = 'Meituan',
   CometAPI = 'CometAPI',
-  DeerAPI = 'DeerAPI',
 }
 
 // Please lowercase the file name
@@ -120,5 +119,4 @@ export const IconMap = {
   [LLMFactory.TokenPony]: 'token-pony',
   [LLMFactory.Meituan]: 'longcat',
   [LLMFactory.CometAPI]: 'cometapi',
-  [LLMFactory.DeerAPI]: 'deerapi',
 };

web/src/hooks/use-agent-request.ts

@@ -3,6 +3,7 @@ import { useHandleFilterSubmit } from '@/components/list-filter-bar/use-handle-f
 import message from '@/components/ui/message';
 import { AgentGlobals } from '@/constants/agent';
 import {
+  DSL,
   IAgentLogsRequest,
   IAgentLogsResponse,
   IFlow,
@@ -294,7 +295,7 @@ export const useSetAgent = (showMessage: boolean = true) => {
     mutationFn: async (params: {
       id?: string;
       title?: string;
-      dsl?: Record<string, any>;
+      dsl?: DSL;
       avatar?: string;
       canvas_category?: string;
     }) => {

web/src/hooks/use-knowledge-request.ts

@@ -32,7 +32,6 @@ export const enum KnowledgeApiAction {
   FetchKnowledgeGraph = 'fetchKnowledgeGraph',
   FetchMetadata = 'fetchMetadata',
   FetchKnowledgeList = 'fetchKnowledgeList',
-  RemoveKnowledgeGraph = 'removeKnowledgeGraph',
 }
 
 export const useKnowledgeBaseId = (): string => {
@@ -144,7 +143,7 @@ export const useFetchNextKnowledgeListByPage = () => {
 
   const onInputChange: React.ChangeEventHandler<HTMLInputElement> = useCallback(
     (e) => {
-      // setPagination({ page: 1 }); // TODO: This results in repeated requests
+      // setPagination({ page: 1 }); // TODO: 这里导致重复请求
       handleInputChange(e);
     },
     [handleInputChange],
@@ -323,7 +322,7 @@ export const useRemoveKnowledgeGraph = () => {
       if (data.code === 0) {
         message.success(i18n.t(`message.deleted`));
         queryClient.invalidateQueries({
-          queryKey: [KnowledgeApiAction.FetchKnowledgeGraph],
+          queryKey: ['fetchKnowledgeGraph'],
         });
       }
       return data?.code;