Fix: Fixed the issue where tag content would overflow the container #8392 (#8393)

### What problem does this PR solve?

Fix: Fixed the issue where tag content would overflow the container
#8392
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

.gitattributes

@@ -1 +1,2 @@
-*.sh text eol=lf
+*.sh text eol=lf
+docker/entrypoint.sh text eol=lf executable

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,15 +1,21 @@
-name: Bug Report
+name: "🐞 Bug Report"
 description: Create a bug issue for RAGFlow
 title: "[Bug]: "
-labels: [bug]
+labels: ["🐞 bug"]
 body:
 - type: checkboxes
   attributes:
-    label: Is there an existing issue for the same bug?
-    description: Please check if an issue already exists for the bug you encountered.
+    label: Self Checks
+    description: "Please check the following in order to be responded in time :)"
     options:
-    - label: I have checked the existing issues.
-      required: true
+      - label: I have searched for existing issues [search for existing issues](https://github.com/infiniflow/ragflow/issues), including closed ones.
+        required: true
+      - label: I confirm that I am using English to submit this report ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+        required: true
+      - label: Non-english title submitions will be closed directly ( 非英文标题的提交将会被直接关闭 ) ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+        required: true
+      - label: "Please do not modify this template :) and fill in all the required fields."
+        required: true
 - type: markdown
   attributes:
     value: "Please provide the following information to help us understand the issue."

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,10 +0,0 @@
----
-name: Feature request
-title: '[Feature Request]: '
-about: Suggest an idea for RAGFlow
-labels: ''
----
-
-**Summary**
-
-Description for this feature.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,14 +1,20 @@
-name: Feature request
+name: "💞 Feature request"
 description: Propose a feature request for RAGFlow.
 title: "[Feature Request]: "
-labels: [feature request]
+labels: ["💞 feature"]
 body:
   - type: checkboxes
     attributes:
-      label: Is there an existing issue for the same feature request?
-      description: Please check if an issue already exists for the feature you request.
+      label: Self Checks
+      description: "Please check the following in order to be responded in time :)"
       options:
-        - label: I have checked the existing issues.
+        - label: I have searched for existing issues [search for existing issues](https://github.com/infiniflow/ragflow/issues), including closed ones.
+          required: true
+        - label: I confirm that I am using English to submit this report ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+          required: true
+        - label: Non-english title submitions will be closed directly ( 非英文标题的提交将会被直接关闭 ) ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+          required: true
+        - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea
     attributes:

.github/ISSUE_TEMPLATE/question.yml

@@ -1,8 +1,21 @@
-name: Question
+name: "🙋‍♀️ Question"
 description: Ask questions on RAGFlow
 title: "[Question]: "
-labels: [question]
+labels: ["🙋‍♀️ question"]
 body:
+- type: checkboxes
+  attributes:
+    label: Self Checks
+    description: "Please check the following in order to be responded in time :)"
+    options:
+      - label: I have searched for existing issues [search for existing issues](https://github.com/infiniflow/ragflow/issues), including closed ones.
+        required: true
+      - label: I confirm that I am using English to submit this report ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+        required: true
+      - label: Non-english title submitions will be closed directly ( 非英文标题的提交将会被直接关闭 ) ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+        required: true
+      - label: "Please do not modify this template :) and fill in all the required fields."
+        required: true
 - type: markdown
   attributes:
     value: |

.github/workflows/release.yml

@@ -75,12 +75,6 @@ jobs:
           # The body field does not support environment variable substitution directly.
           body_path: release_body.md
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
       # https://github.com/marketplace/actions/docker-login
       - name: Login to Docker Hub
         uses: docker/login-action@v3

.github/workflows/tests.yml

@@ -15,6 +15,8 @@ on:
       - 'docs/**'
       - '*.md'
       - '*.mdx'
+  schedule:
+    - cron: '0 16 * * *'  # This schedule runs every 16:00:00Z(00:00:00+08:00)
 
 # https://docs.github.com/en/actions/using-jobs/using-concurrency
 concurrency:
@@ -32,12 +34,9 @@ jobs:
       # https://github.com/hmarr/debug-action
       #- uses: hmarr/debug-action@v2
 
-      - name: Show PR labels
+      - name: Show who triggered this workflow
         run: |
           echo "Workflow triggered by ${{ github.event_name }}"
-          if [[ ${{ github.event_name }} == 'pull_request' ]]; then
-            echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}"
-          fi
 
       - name: Ensure workspace ownership
         run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
@@ -51,10 +50,10 @@ jobs:
 
       # https://github.com/astral-sh/ruff-action
       - name: Static check with Ruff
-        uses: astral-sh/ruff-action@v2
+        uses: astral-sh/ruff-action@v3
         with:
-          version: ">=0.8.2"
-          args: "check --ignore E402"
+          version: ">=0.11.x"
+          args: "check"
 
       - name: Build ragflow:nightly-slim
         run: |
@@ -68,7 +67,7 @@ jobs:
 
       - name: Start ragflow:nightly-slim
         run: |
-          echo "RAGFLOW_IMAGE=infiniflow/ragflow:nightly-slim" >> docker/.env
+          echo -e "\nRAGFLOW_IMAGE=infiniflow/ragflow:nightly-slim" >> docker/.env
           sudo docker compose -f docker/docker-compose.yml up -d
 
       - name: Stop ragflow:nightly-slim
@@ -78,7 +77,7 @@ jobs:
 
       - name: Start ragflow:nightly
         run: |
-          echo "RAGFLOW_IMAGE=infiniflow/ragflow:nightly" >> docker/.env
+          echo -e "\nRAGFLOW_IMAGE=infiniflow/ragflow:nightly" >> docker/.env
           sudo docker compose -f docker/docker-compose.yml up -d
 
       - name: Run sdk tests against Elasticsearch
@@ -89,7 +88,12 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && uv sync --python 3.10 --frozen && uv pip install . && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python && uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
 
       - name: Run frontend api tests against Elasticsearch
         run: |
@@ -99,8 +103,22 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && uv sync --python 3.10 --frozen && uv pip install . && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py 
+          cd sdk/python && UV_LINK_MODE=copy uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
           
+      - name: Run http api tests against Elasticsearch
+        run: |
+          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
+          export HOST_ADDRESS=http://host.docker.internal:9380
+          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+            echo "Waiting for service to be available..."
+            sleep 5
+          done
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
@@ -119,7 +137,12 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && uv sync --python 3.10 --frozen && uv pip install . && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python && DOC_ENGINE=infinity uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
 
       - name: Run frontend api tests against Infinity
         run: |
@@ -129,7 +152,22 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && uv sync --python 3.10 --frozen && uv pip install . && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
+          cd sdk/python && UV_LINK_MODE=copy uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
+
+      - name: Run http api tests against Infinity
+        run: |
+          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
+          export HOST_ADDRESS=http://host.docker.internal:9380
+          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+            echo "Waiting for service to be available..."
+            sleep 5
+          done
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && DOC_ENGINE=infinity uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed

.gitignore

@@ -36,5 +36,160 @@ sdk/python/ragflow.egg-info/
 sdk/python/build/
 sdk/python/dist/
 sdk/python/ragflow_sdk.egg-info/
+
+# Exclude dep files
+libssl*.deb
+tika-server*.jar*
+cl100k_base.tiktoken
+chrome*
 huggingface.co/
 nltk_data/
+
+# Exclude hash-like temporary files like 9b5ad71b2ce5302211f9c61530b329a4922fc6a4
+*[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*
+.lh/
+.venv
+docker/data
+
+
+#--------------------------------------------------#
+# The following was generated with gitignore.nvim: #
+#--------------------------------------------------#
+# Gitignore for the following technologies: Node
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# Serverless Webpack directories
+.webpack/
+
+# SvelteKit build / generate output
+.svelte-kit
+

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-yaml
+      - id: check-json
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: mixed-line-ending
+      - id: check-symlinks
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.6
+    hooks:
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format

Dockerfile

@@ -21,9 +21,7 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co
     if [ "$LIGHTEN" != "1" ]; then \
         (tar -cf - \
             /huggingface.co/BAAI/bge-large-zh-v1.5 \
-            /huggingface.co/BAAI/bge-reranker-v2-m3 \
             /huggingface.co/maidalun1020/bce-embedding-base_v1 \
-            /huggingface.co/maidalun1020/bce-reranker-base_v1 \
             | tar -xf - --strip-components=2 -C /root/.ragflow) \
     fi
 
@@ -46,7 +44,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 # Building C extensions: libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev
 RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     if [ "$NEED_MIRROR" == "1" ]; then \
-        sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list; \
+        sed -i 's|http://ports.ubuntu.com|http://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list; \
+        sed -i 's|http://archive.ubuntu.com|http://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list; \
     fi; \
     rm -f /etc/apt/apt.conf.d/docker-clean && \
     echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache && \
@@ -59,14 +58,16 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     apt install -y default-jdk && \
     apt install -y libatk-bridge2.0-0 && \
     apt install -y libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev && \
-    apt install -y python3-pip pipx nginx unzip curl wget git vim less
+    apt install -y libjemalloc-dev && \
+    apt install -y python3-pip pipx nginx unzip curl wget git vim less && \
+    apt install -y ghostscript
 
 RUN if [ "$NEED_MIRROR" == "1" ]; then \
-        pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \
-        pip3 config set global.trusted-host pypi.tuna.tsinghua.edu.cn; \
+        pip3 config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
+        pip3 config set global.trusted-host mirrors.aliyun.com; \
         mkdir -p /etc/uv && \
         echo "[[index]]" > /etc/uv/uv.toml && \
-        echo 'url = "https://pypi.tuna.tsinghua.edu.cn/simple"' >> /etc/uv/uv.toml && \
+        echo 'url = "https://mirrors.aliyun.com/pypi/simple"' >> /etc/uv/uv.toml && \
         echo "default = true" >> /etc/uv/uv.toml; \
     fi; \
     pipx install uv
@@ -150,9 +151,9 @@ COPY pyproject.toml uv.lock ./
 # uv records index url into uv.lock but doesn't failover among multiple indexes
 RUN --mount=type=cache,id=ragflow_uv,target=/root/.cache/uv,sharing=locked \
     if [ "$NEED_MIRROR" == "1" ]; then \
-        sed -i 's|pypi.org|pypi.tuna.tsinghua.edu.cn|g' uv.lock; \
+        sed -i 's|pypi.org|mirrors.aliyun.com/pypi|g' uv.lock; \
     else \
-        sed -i 's|pypi.tuna.tsinghua.edu.cn|pypi.org|g' uv.lock; \
+        sed -i 's|mirrors.aliyun.com/pypi|pypi.org|g' uv.lock; \
     fi; \
     if [ "$LIGHTEN" == "1" ]; then \
         uv sync --python 3.10 --frozen; \
@@ -196,10 +197,13 @@ COPY deepdoc deepdoc
 COPY rag rag
 COPY agent agent
 COPY graphrag graphrag
+COPY agentic_reasoning agentic_reasoning
 COPY pyproject.toml uv.lock ./
+COPY mcp mcp
+COPY plugin plugin
 
 COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template
-COPY docker/entrypoint.sh docker/entrypoint-parser.sh ./
+COPY docker/entrypoint.sh ./
 RUN chmod +x ./entrypoint*.sh
 
 # Copy compiled web pages

Dockerfile.scratch.oc9

@@ -33,6 +33,7 @@ ADD ./rag ./rag
 ADD ./requirements.txt ./requirements.txt
 ADD ./agent ./agent
 ADD ./graphrag ./graphrag
+ADD ./plugin ./plugin
 
 RUN dnf install -y openmpi openmpi-devel python3-openmpi
 ENV C_INCLUDE_PATH /usr/include/openmpi-x86_64:$C_INCLUDE_PATH

README.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DBEDFA"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.16.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.16.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,16 +30,25 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
+
 <details open>
 <summary><b>📕 Table of Contents</b></summary>
 
@@ -78,12 +87,11 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Latest Updates
 
-- 2025-02-05 Updates the model list of 'SILICONFLOW' and adds support for Deepseek-R1/DeepSeek-V3.
-- 2025-01-26 Optimizes knowledge graph extraction and application, offering various configuration options.
-- 2024-12-18 Upgrades Document Layout Analysis model in Deepdoc.
-- 2024-12-04 Adds support for pagerank score in knowledge base.
-- 2024-11-22 Adds more variables to Agent.
-- 2024-11-01 Adds keyword extraction and related question generation to the parsed chunks to improve the accuracy of retrieval.
+- 2025-05-23 Adds a Python/JavaScript code executor component to Agent.
+- 2025-05-05 Supports cross-language query.
+- 2025-03-19 Supports using a multi-modal model to make sense of images within PDF or DOCX files.
+- 2025-02-28 Combined with Internet search (Tavily), supports reasoning like Deep Research for any LLMs.
+- 2024-12-18 Upgrades Document Layout Analysis model in DeepDoc.
 - 2024-08-22 Support text to SQL statements through RAG.
 
 ## 🎉 Stay Tuned
@@ -138,8 +146,10 @@ releases! 🌟
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > If you have not installed Docker on your local machine (Windows, Mac, or Linux),
-  > see [Install Docker Engine](https://docs.docker.com/engine/install/).
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): Required only if you intend to use the code executor (sandbox) feature of RAGFlow.
+
+> [!TIP]
+> If you have not installed Docker on your local machine (Windows, Mac, or Linux), see [Install Docker Engine](https://docs.docker.com/engine/install/).
 
 ### 🚀 Start up the server
 
@@ -173,19 +183,27 @@ releases! 🌟
 
 3. Start up the server using the pre-built Docker images:
 
-   > The command below downloads the `v0.16.0-slim` edition of the RAGFlow Docker image. Refer to the following table for descriptions of different RAGFlow editions. To download an RAGFlow edition different from `v0.16.0-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.16.0` for the full edition `v0.16.0`.
+> [!CAUTION]
+> All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
+> If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
+
+   > The command below downloads the `v0.19.1-slim` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.19.1-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` for the full edition `v0.19.1`.
 
    ```bash
-   $ cd ragflow
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
    ```
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    |-------------------|-----------------|-----------------------|--------------------------|
-   | v0.16.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.16.0-slim      | &approx;2       | ❌                     | Stable release           |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                   | Stable release            |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                   | _Unstable_ nightly build  |
 
 4. Check the server status after having the server up and running:
 
@@ -204,9 +222,6 @@ releases! 🌟
      /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
 
    > If you skip this confirmation step and directly log in to RAGFlow, your browser may prompt a `network anormal`
@@ -240,7 +255,7 @@ to `<YOUR_SERVING_PORT>:80`.
 Updates to the above configurations require a reboot of all containers to take effect:
 
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ### Switch doc engine from Elasticsearch to Infinity
@@ -253,12 +268,15 @@ RAGFlow uses Elasticsearch by default for storing full text and vectors. To swit
    $ docker compose -f docker/docker-compose.yml down -v
    ```
 
+> [!WARNING]
+> `-v` will delete the docker container volumes, and the existing data will be cleared.
+
 2. Set `DOC_ENGINE` in **docker/.env** to `infinity`.
 
 3. Start the containers:
 
    ```bash
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
 
 > [!WARNING]
@@ -271,7 +289,7 @@ This image is approximately 2 GB in size and relies on external LLM and embeddin
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 Build a Docker image including embedding models
@@ -281,7 +299,7 @@ This image is approximately 9 GB in size. As it includes embedding models, it re
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 Launch service from source for development
@@ -289,7 +307,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 1. Install uv, or skip this step if it is already installed:
 
    ```bash
-   pipx install uv
+   pipx install uv pre-commit
    ```
 
 2. Clone the source code and install Python dependencies:
@@ -298,6 +316,8 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
    uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. Launch the dependent services (MinIO, Elasticsearch, Redis, and MySQL) using Docker Compose:
@@ -309,7 +329,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    Add the following line to `/etc/hosts` to resolve all hosts specified in **docker/.env** to `127.0.0.1`:
 
    ```
-   127.0.0.1       es01 infinity mysql minio redis
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
 
 4. If you cannot access HuggingFace, set the `HF_ENDPOINT` environment variable to use a mirror site:
@@ -318,7 +338,16 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. Launch backend service:
+5. If your operating system does not have jemalloc, please install it as follows:
+
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+   
+6. Launch backend service:
 
    ```bash
    source .venv/bin/activate
@@ -326,12 +355,14 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    bash docker/launch_backend_service.sh
    ```
 
-6. Install frontend dependencies:
+7. Install frontend dependencies:
+
    ```bash
    cd web
    npm install
    ```
-7. Launch frontend service:
+
+8. Launch frontend service:
 
    ```bash
    npm run dev
@@ -341,12 +372,22 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+9. Stop RAGFlow front-end and back-end service after development is complete:
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
 ## 📚 Documentation
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/guides)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
 - [References](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 Roadmap
 
@@ -354,11 +395,11 @@ See the [RAGFlow Roadmap 2025](https://github.com/infiniflow/ragflow/issues/4214
 
 ## 🏄 Community
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/NjYzJD3GM3)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Contributing
 
 RAGFlow flourishes via open-source collaboration. In this spirit, we embrace diverse contributions from the community.
-If you would like to be a part, review our [Contribution Guidelines](./CONTRIBUTING.md) first.
+If you would like to be a part, review our [Contribution Guidelines](https://ragflow.io/docs/dev/contributing) first.

README_id.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體中文版自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DBEDFA"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Lencana Daring" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.16.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.16.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Rilis%20Terbaru" alt="Rilis Terbaru">
@@ -30,18 +30,23 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/Lisensi-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="Lisensi">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Dokumentasi</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/4214">Peta Jalan</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
 <details open>
-<summary></b>📕 Daftar Isi</b></summary>
+<summary><b>📕 Daftar Isi </b> </summary>
 
 - 💡 [Apa Itu RAGFlow?](#-apa-itu-ragflow)
 - 🎮 [Demo](#-demo)
@@ -75,12 +80,11 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Pembaruan Terbaru
 
-- 2025-02-05 Memperbarui daftar model 'SILICONFLOW' dan menambahkan dukungan untuk Deepseek-R1/DeepSeek-V3.
-- 2025-01-26 Optimalkan ekstraksi dan penerapan grafik pengetahuan dan sediakan berbagai opsi konfigurasi.
-- 2024-12-18 Meningkatkan model Analisis Tata Letak Dokumen di Deepdoc.
-- 2024-12-04 Mendukung skor pagerank ke basis pengetahuan.
-- 2024-11-22 Peningkatan definisi dan penggunaan variabel di Agen.
-- 2024-11-01 Penambahan ekstraksi kata kunci dan pembuatan pertanyaan terkait untuk meningkatkan akurasi pengambilan.
+- 2025-05-23 Menambahkan komponen pelaksana kode Python/JS ke Agen.
+- 2025-05-05 Mendukung kueri lintas bahasa.
+- 2025-03-19 Mendukung penggunaan model multi-modal untuk memahami gambar di dalam file PDF atau DOCX.
+- 2025-02-28 dikombinasikan dengan pencarian Internet (TAVILY), mendukung penelitian mendalam untuk LLM apa pun.
+- 2024-12-18 Meningkatkan model Analisis Tata Letak Dokumen di DeepDoc.
 - 2024-08-22 Dukungan untuk teks ke pernyataan SQL melalui RAG.
 
 ## 🎉 Tetap Terkini
@@ -133,6 +137,10 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): Hanya diperlukan jika Anda ingin menggunakan fitur eksekutor kode (sandbox) dari RAGFlow.
+
+> [!TIP]
+> Jika Anda belum menginstal Docker di komputer lokal Anda (Windows, Mac, atau Linux), lihat [Install Docker Engine](https://docs.docker.com/engine/install/).
 
 ### 🚀 Menjalankan Server
 
@@ -166,21 +174,29 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
 3. Bangun image Docker pre-built dan jalankan server:
 
-   > Perintah di bawah ini mengunduh edisi v0.16.0-slim dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.16.0-slim, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server. Misalnya, atur RAGFLOW_IMAGE=infiniflow/ragflow:v0.16.0 untuk edisi lengkap v0.16.0.
+> [!CAUTION]
+> Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
+> Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
 
-   ```bash
-   $ cd ragflow
-   $ docker compose -f docker/docker-compose.yml up -d
-   ```
+> Perintah di bawah ini mengunduh edisi v0.19.1-slim dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.19.1-slim, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server. Misalnya, atur RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1 untuk edisi lengkap v0.19.1.
 
-   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-   | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.16.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.16.0-slim      | &approx;2       | ❌                    | Stable release           |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
+```bash
+$ cd ragflow/docker
+# Use CPU for embedding and DeepDoc tasks:
+$ docker compose -f docker-compose.yml up -d
 
-4. Periksa status server setelah server aktif dan berjalan:
+# To use GPU to accelerate embedding and DeepDoc tasks:
+# docker compose -f docker-compose-gpu.yml up -d
+```
+
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+| ----------------- | --------------- | --------------------- | ------------------------ |
+| v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+| v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
+| nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
+| nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
+
+1. Periksa status server setelah server aktif dan berjalan:
 
    ```bash
    $ docker logs -f ragflow-server
@@ -197,18 +213,15 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
      /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
 
    > Jika Anda melewatkan langkah ini dan langsung login ke RAGFlow, browser Anda mungkin menampilkan error `network anormal`
    > karena RAGFlow mungkin belum sepenuhnya siap.
 
-5. Buka browser web Anda, masukkan alamat IP server Anda, dan login ke RAGFlow.
+2. Buka browser web Anda, masukkan alamat IP server Anda, dan login ke RAGFlow.
    > Dengan pengaturan default, Anda hanya perlu memasukkan `http://IP_DEVICE_ANDA` (**tanpa** nomor port) karena
    > port HTTP default `80` bisa dihilangkan saat menggunakan konfigurasi default.
-6. Dalam [service_conf.yaml.template](./docker/service_conf.yaml.template), pilih LLM factory yang diinginkan di `user_default_llm` dan perbarui
+3. Dalam [service_conf.yaml.template](./docker/service_conf.yaml.template), pilih LLM factory yang diinginkan di `user_default_llm` dan perbarui
    bidang `API_KEY` dengan kunci API yang sesuai.
 
    > Lihat [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) untuk informasi lebih lanjut.
@@ -230,7 +243,7 @@ menjadi `<YOUR_SERVING_PORT>:80`.
 Pembaruan konfigurasi ini memerlukan reboot semua kontainer agar efektif:
 
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ## 🔧 Membangun Docker Image tanpa Model Embedding
@@ -240,7 +253,7 @@ Image ini berukuran sekitar 2 GB dan bergantung pada aplikasi LLM eksternal dan
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 Membangun Docker Image Termasuk Model Embedding
@@ -250,7 +263,7 @@ Image ini berukuran sekitar 9 GB. Karena sudah termasuk model embedding, ia hany
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 Menjalankan Aplikasi dari untuk Pengembangan
@@ -258,7 +271,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 1. Instal uv, atau lewati langkah ini jika sudah terinstal:
 
    ```bash
-   pipx install uv
+   pipx install uv pre-commit
    ```
 
 2. Clone kode sumber dan instal dependensi Python:
@@ -267,6 +280,8 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
    uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. Jalankan aplikasi yang diperlukan (MinIO, Elasticsearch, Redis, dan MySQL) menggunakan Docker Compose:
@@ -278,7 +293,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    Tambahkan baris berikut ke `/etc/hosts` untuk memetakan semua host yang ditentukan di **conf/service_conf.yaml** ke `127.0.0.1`:
 
    ```
-   127.0.0.1       es01 infinity mysql minio redis
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
 
 4. Jika Anda tidak dapat mengakses HuggingFace, atur variabel lingkungan `HF_ENDPOINT` untuk menggunakan situs mirror:
@@ -287,7 +302,16 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. Jalankan aplikasi backend:
+5. Jika sistem operasi Anda tidak memiliki jemalloc, instal sebagai berikut:
+
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+
+6. Jalankan aplikasi backend:
 
    ```bash
    source .venv/bin/activate
@@ -295,12 +319,14 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    bash docker/launch_backend_service.sh
    ```
 
-6. Instal dependensi frontend:
+7. Instal dependensi frontend:
+
    ```bash
    cd web
    npm install
    ```
-7. Jalankan aplikasi frontend:
+
+8. Jalankan aplikasi frontend:
 
    ```bash
    npm run dev
@@ -310,12 +336,23 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+
+9. Hentikan layanan front-end dan back-end RAGFlow setelah pengembangan selesai:
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
 ## 📚 Dokumentasi
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [Panduan Pengguna](https://ragflow.io/docs/dev/category/guides)
-- [Referensi](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
+- [References](https://ragflow.io/docs/dev/category/references)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 Roadmap
 
@@ -323,11 +360,11 @@ Lihat [Roadmap RAGFlow 2025](https://github.com/infiniflow/ragflow/issues/4214)
 
 ## 🏄 Komunitas
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/NjYzJD3GM3)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Kontribusi
 
 RAGFlow berkembang melalui kolaborasi open-source. Dalam semangat ini, kami menerima kontribusi dari komunitas.
-Jika Anda ingin berpartisipasi, tinjau terlebih dahulu [Panduan Kontribusi](./CONTRIBUTING.md).
+Jika Anda ingin berpartisipasi, tinjau terlebih dahulu [Panduan Kontribusi](https://ragflow.io/docs/dev/contributing).

README_ja.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體中文版自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DBEDFA"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.16.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.16.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,16 +30,21 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
 ## 💡 RAGFlow とは？
 
 [RAGFlow](https://ragflow.io/) は、深い文書理解に基づいたオープンソースの RAG (Retrieval-Augmented Generation) エンジンである。LLM（大規模言語モデル）を組み合わせることで、様々な複雑なフォーマットのデータから根拠のある引用に裏打ちされた、信頼できる質問応答機能を実現し、あらゆる規模のビジネスに適した RAG ワークフローを提供します。
@@ -55,12 +60,11 @@
 
 ## 🔥 最新情報
 
-- 2025-02-05 シリコン フローの St およびモデル リストを更新し、Deep Seek-R1/Deep Seek-V3 のサポートを追加しました。
-- 2025-01-26 ナレッジ グラフの抽出と適用を最適化し、さまざまな構成オプションを提供します。
-- 2024-12-18 Deepdoc のドキュメント レイアウト分析モデルをアップグレードします。
-- 2024-12-04 ナレッジ ベースへのページランク スコアをサポートしました。
-- 2024-11-22 エージェントでの変数の定義と使用法を改善しました。
-- 2024-11-01 再現の精度を向上させるために、解析されたチャンクにキーワード抽出と関連質問の生成を追加しました。
+- 2025-05-23 エージェントに Python/JS コードエグゼキュータコンポーネントを追加しました。
+- 2025-05-05 言語間クエリをサポートしました。
+- 2025-03-19 PDFまたはDOCXファイル内の画像を理解するために、多モーダルモデルを使用することをサポートします。
+- 2025-02-28 インターネット検索 (TAVILY) と組み合わせて、あらゆる LLM の詳細な調査をサポートします。
+- 2024-12-18 DeepDoc のドキュメント レイアウト分析モデルをアップグレードします。
 - 2024-08-22 RAG を介して SQL ステートメントへのテキストをサポートします。
 
 ## 🎉 続きを楽しみに
@@ -113,7 +117,10 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > ローカルマシン（Windows、Mac、または Linux）に Docker をインストールしていない場合は、[Docker Engine のインストール](https://docs.docker.com/engine/install/) を参照してください。
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): RAGFlowのコード実行（サンドボックス）機能を利用する場合のみ必要です。
+
+> [!TIP]
+> ローカルマシン（Windows、Mac、または Linux）に Docker をインストールしていない場合は、[Docker Engine のインストール](https://docs.docker.com/engine/install/) を参照してください。
 
 ### 🚀 サーバーを起動
 
@@ -146,21 +153,29 @@
 
 3. ビルド済みの Docker イメージをビルドし、サーバーを起動する:
 
-   > 以下のコマンドは、RAGFlow Docker イメージの v0.16.0-slim エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.16.0-slim とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。例えば、完全版 v0.16.0 をダウンロードするには、RAGFLOW_IMAGE=infiniflow/ragflow:v0.16.0 と設定します。
+> [!CAUTION]
+> 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
+> ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
+
+   > 以下のコマンドは、RAGFlow Docker イメージの v0.19.1-slim エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.19.1-slim とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。例えば、完全版 v0.19.1 をダウンロードするには、RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1 と設定します。
 
    ```bash
-   $ cd ragflow
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
    ```
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.16.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.16.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 
-4. サーバーを立ち上げた後、サーバーの状態を確認する:
+1. サーバーを立ち上げた後、サーバーの状態を確認する:
 
    ```bash
    $ docker logs -f ragflow-server
@@ -176,16 +191,13 @@
     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
 
    > もし確認ステップをスキップして直接 RAGFlow にログインした場合、その時点で RAGFlow が完全に初期化されていない可能性があるため、ブラウザーがネットワーク異常エラーを表示するかもしれません。
 
-5. ウェブブラウザで、プロンプトに従ってサーバーの IP アドレスを入力し、RAGFlow にログインします。
+2. ウェブブラウザで、プロンプトに従ってサーバーの IP アドレスを入力し、RAGFlow にログインします。
    > デフォルトの設定を使用する場合、デフォルトの HTTP サービングポート `80` は省略できるので、与えられたシナリオでは、`http://IP_OF_YOUR_MACHINE`（ポート番号は省略）だけを入力すればよい。
-6. [service_conf.yaml.template](./docker/service_conf.yaml.template) で、`user_default_llm` で希望の LLM ファクトリを選択し、`API_KEY` フィールドを対応する API キーで更新する。
+3. [service_conf.yaml.template](./docker/service_conf.yaml.template) で、`user_default_llm` で希望の LLM ファクトリを選択し、`API_KEY` フィールドを対応する API キーで更新する。
 
    > 詳しくは [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) を参照してください。
 
@@ -208,7 +220,7 @@
 > すべてのシステム設定のアップデートを有効にするには、システムの再起動が必要です:
 >
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ### Elasticsearch から Infinity にドキュメントエンジンを切り替えます
@@ -219,13 +231,14 @@ RAGFlow はデフォルトで Elasticsearch を使用して全文とベクトル
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+   Note: `-v` は docker コンテナのボリュームを削除し、既存のデータをクリアします。
 2. **docker/.env** の「DOC \_ ENGINE」を「infinity」に設定します。
 
 3. 起動コンテナ：
    ```bash
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
-   > [!WARNING]  
+   > [!WARNING]
    > Linux/arm64 マシンでの Infinity への切り替えは正式にサポートされていません。
 
 ## 🔧 ソースコードで Docker イメージを作成（埋め込みモデルなし）
@@ -235,7 +248,7 @@ RAGFlow はデフォルトで Elasticsearch を使用して全文とベクトル
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 ソースコードをコンパイルした Docker イメージ（埋め込みモデルを含む）
@@ -245,7 +258,7 @@ docker build --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-s
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 ソースコードからサービスを起動する方法
@@ -253,7 +266,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 1. uv をインストールする。すでにインストールされている場合は、このステップをスキップしてください:
 
    ```bash
-   pipx install uv
+   pipx install uv pre-commit
    ```
 
 2. ソースコードをクローンし、Python の依存関係をインストールする:
@@ -262,6 +275,8 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
    uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. Docker Compose を使用して依存サービス（MinIO、Elasticsearch、Redis、MySQL）を起動する:
@@ -273,7 +288,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    `/etc/hosts` に以下の行を追加して、**conf/service_conf.yaml** に指定されたすべてのホストを `127.0.0.1` に解決します:
 
    ```
-   127.0.0.1       es01 infinity mysql minio redis
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
 
 4. HuggingFace にアクセスできない場合は、`HF_ENDPOINT` 環境変数を設定してミラーサイトを使用してください:
@@ -282,7 +297,16 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. バックエンドサービスを起動する:
+5. オペレーティングシステムにjemallocがない場合は、次のようにインストールします:
+   
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+
+6. バックエンドサービスを起動する:
 
    ```bash
    source .venv/bin/activate
@@ -290,12 +314,14 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    bash docker/launch_backend_service.sh
    ```
 
-6. フロントエンドの依存関係をインストールする:
+7. フロントエンドの依存関係をインストールする:
+
    ```bash
    cd web
    npm install
    ```
-7. フロントエンドサービスを起動する:
+
+8. フロントエンドサービスを起動する:
 
    ```bash
    npm run dev
@@ -305,12 +331,22 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+9. 開発が完了したら、RAGFlow のフロントエンド サービスとバックエンド サービスを停止します:
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
 ## 📚 ドキュメンテーション
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/guides)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
 - [References](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 ロードマップ
 
@@ -318,10 +354,10 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 
 ## 🏄 コミュニティ
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/NjYzJD3GM3)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 コントリビュート
 
-RAGFlow はオープンソースのコラボレーションによって発展してきました。この精神に基づき、私たちはコミュニティからの多様なコントリビュートを受け入れています。 参加を希望される方は、まず [コントリビューションガイド](./CONTRIBUTING.md)をご覧ください。
+RAGFlow はオープンソースのコラボレーションによって発展してきました。この精神に基づき、私たちはコミュニティからの多様なコントリビュートを受け入れています。 参加を希望される方は、まず [コントリビューションガイド](https://ragflow.io/docs/dev/contributing)をご覧ください。

README_ko.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DBEDFA"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.16.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.16.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,16 +30,21 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
 ## 💡 RAGFlow란?
 
 [RAGFlow](https://ragflow.io/)는 심층 문서 이해에 기반한 오픈소스 RAG (Retrieval-Augmented Generation) 엔진입니다. 이 엔진은 대규모 언어 모델(LLM)과 결합하여 정확한 질문 응답 기능을 제공하며, 다양한 복잡한 형식의 데이터에서 신뢰할 수 있는 출처를 바탕으로 한 인용을 통해 이를 뒷받침합니다. RAGFlow는 규모에 상관없이 모든 기업에 최적화된 RAG 워크플로우를 제공합니다.
@@ -55,13 +60,11 @@
 
 ## 🔥 업데이트
 
-- 2025-02-05 'SILICONFLOW' 모델 목록을 업데이트하고 Deepseek-R1/DeepSeek-V3에 대한 지원을 추가합니다.
-- 2025-01-26 지식 그래프 추출 및 적용을 최적화하고 다양한 구성 옵션을 제공합니다.
-- 2024-12-18 Deepdoc의 문서 레이아웃 분석 모델 업그레이드.
-- 2024-12-04 지식베이스에 대한 페이지랭크 점수를 지원합니다.
-
-- 2024-11-22 에이전트의 변수 정의 및 사용을 개선했습니다.
-- 2024-11-01 파싱된 청크에 키워드 추출 및 관련 질문 생성을 추가하여 재현율을 향상시킵니다.
+- 2025-05-23 Agent에 Python/JS 코드 실행기 구성 요소를 추가합니다.
+- 2025-05-05 언어 간 쿼리를 지원합니다.
+- 2025-03-19 PDF 또는 DOCX 파일 내의 이미지를 이해하기 위해 다중 모드 모델을 사용하는 것을 지원합니다.
+- 2025-02-28 인터넷 검색(TAVILY)과 결합되어 모든 LLM에 대한 심층 연구를 지원합니다.
+- 2024-12-18 DeepDoc의 문서 레이아웃 분석 모델 업그레이드.
 - 2024-08-22 RAG를 통해 SQL 문에 텍스트를 지원합니다.
 
 ## 🎉 계속 지켜봐 주세요
@@ -114,7 +117,10 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > 로컬 머신(Windows, Mac, Linux)에 Docker가 설치되지 않은 경우, [Docker 엔진 설치](<(https://docs.docker.com/engine/install/)>)를 참조하세요.
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): RAGFlow의 코드 실행기(샌드박스) 기능을 사용하려는 경우에만 필요합니다.
+
+> [!TIP]
+> 로컬 머신(Windows, Mac, Linux)에 Docker가 설치되지 않은 경우, [Docker 엔진 설치](<(https://docs.docker.com/engine/install/)>)를 참조하세요.
 
 ### 🚀 서버 시작하기
 
@@ -147,21 +153,29 @@
 
 3. 미리 빌드된 Docker 이미지를 생성하고 서버를 시작하세요:
 
-   > 아래 명령어는 RAGFlow Docker 이미지의 v0.16.0-slim 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.16.0-slim과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오. 예를 들어, 전체 버전인 v0.16.0을 다운로드하려면 RAGFLOW_IMAGE=infiniflow/ragflow:v0.16.0로 설정합니다.
+> [!CAUTION]
+> 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
+> ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
+
+   > 아래 명령어는 RAGFlow Docker 이미지의 v0.19.1-slim 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.19.1-slim과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오. 예를 들어, 전체 버전인 v0.19.1을 다운로드하려면 RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1로 설정합니다.
 
    ```bash
-   $ cd ragflow
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
    ```
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.16.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.16.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 
-4. 서버가 시작된 후 서버 상태를 확인하세요:
+1. 서버가 시작된 후 서버 상태를 확인하세요:
 
    ```bash
    $ docker logs -f ragflow-server
@@ -177,16 +191,13 @@
     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
 
    > 만약 확인 단계를 건너뛰고 바로 RAGFlow에 로그인하면, RAGFlow가 완전히 초기화되지 않았기 때문에 브라우저에서 `network anormal` 오류가 발생할 수 있습니다.
 
-5. 웹 브라우저에 서버의 IP 주소를 입력하고 RAGFlow에 로그인하세요.
+2. 웹 브라우저에 서버의 IP 주소를 입력하고 RAGFlow에 로그인하세요.
    > 기본 설정을 사용할 경우, `http://IP_OF_YOUR_MACHINE`만 입력하면 됩니다 (포트 번호는 제외). 기본 HTTP 서비스 포트 `80`은 기본 구성으로 사용할 때 생략할 수 있습니다.
-6. [service_conf.yaml.template](./docker/service_conf.yaml.template) 파일에서 원하는 LLM 팩토리를 `user_default_llm`에 선택하고, `API_KEY` 필드를 해당 API 키로 업데이트하세요.
+3. [service_conf.yaml.template](./docker/service_conf.yaml.template) 파일에서 원하는 LLM 팩토리를 `user_default_llm`에 선택하고, `API_KEY` 필드를 해당 API 키로 업데이트하세요.
 
    > 자세한 내용은 [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup)를 참조하세요.
 
@@ -209,7 +220,7 @@
 > 모든 시스템 구성 업데이트는 적용되기 위해 시스템 재부팅이 필요합니다.
 >
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ### Elasticsearch 에서 Infinity 로 문서 엔진 전환
@@ -220,6 +231,7 @@ RAGFlow 는 기본적으로 Elasticsearch 를 사용하여 전체 텍스트 및
    ```bash
    $docker compose-f docker/docker-compose.yml down -v
    ```
+   Note: `-v` 는 docker 컨테이너의 볼륨을 삭제하고 기존 데이터를 지우며, 이 작업은 컨테이너를 중지하는 것과 동일합니다.
 2. **docker/.env**의 "DOC_ENGINE" 을 "infinity" 로 설정합니다.
 3. 컨테이너 부팅:
    ```bash
@@ -235,7 +247,7 @@ RAGFlow 는 기본적으로 Elasticsearch 를 사용하여 전체 텍스트 및
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함)
@@ -245,7 +257,7 @@ docker build --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-s
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 소스 코드로 서비스를 시작합니다.
@@ -253,7 +265,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 1. uv를 설치하거나 이미 설치된 경우 이 단계를 건너뜁니다:
 
    ```bash
-   pipx install uv
+   pipx install uv pre-commit
    ```
 
 2. 소스 코드를 클론하고 Python 의존성을 설치합니다:
@@ -262,6 +274,8 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
    uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. Docker Compose를 사용하여 의존 서비스(MinIO, Elasticsearch, Redis 및 MySQL)를 시작합니다:
@@ -273,7 +287,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    `/etc/hosts` 에 다음 줄을 추가하여 **conf/service_conf.yaml** 에 지정된 모든 호스트를 `127.0.0.1` 로 해결합니다:
 
    ```
-   127.0.0.1       es01 infinity mysql minio redis
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
 
 4. HuggingFace에 접근할 수 없는 경우, `HF_ENDPOINT` 환경 변수를 설정하여 미러 사이트를 사용하세요:
@@ -282,7 +296,16 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. 백엔드 서비스를 시작합니다:
+5. 만약 운영 체제에 jemalloc이 없으면 다음 방식으로 설치하세요:
+
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+
+6. 백엔드 서비스를 시작합니다:
 
    ```bash
    source .venv/bin/activate
@@ -290,12 +313,14 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    bash docker/launch_backend_service.sh
    ```
 
-6. 프론트엔드 의존성을 설치합니다:
+7. 프론트엔드 의존성을 설치합니다:
+
    ```bash
    cd web
    npm install
    ```
-7. 프론트엔드 서비스를 시작합니다:
+
+8. 프론트엔드 서비스를 시작합니다:
 
    ```bash
    npm run dev
@@ -305,12 +330,23 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+
+9. 개발이 완료된 후 RAGFlow 프론트엔드 및 백엔드 서비스를 중지합니다.
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+   
+
 ## 📚 문서
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/guides)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
 - [References](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 로드맵
 
@@ -318,10 +354,10 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 
 ## 🏄 커뮤니티
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/NjYzJD3GM3)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 컨트리뷰션
 
-RAGFlow는 오픈소스 협업을 통해 발전합니다. 이러한 정신을 바탕으로, 우리는 커뮤니티의 다양한 기여를 환영합니다. 참여하고 싶으시다면, 먼저 [가이드라인](./CONTRIBUTING.md)을 검토해 주세요.
+RAGFlow는 오픈소스 협업을 통해 발전합니다. 이러한 정신을 바탕으로, 우리는 커뮤니티의 다양한 기여를 환영합니다. 참여하고 싶으시다면, 먼저 [가이드라인](https://ragflow.io/docs/dev/contributing)을 검토해 주세요.

README_pt_br.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DBEDFA"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Badge Estático" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.16.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.16.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Última%20Relese" alt="Última Versão">
@@ -30,18 +30,23 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="licença">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Documentação</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
 <details open>
-<summary></b>📕 Índice</b></summary>
+<summary><b>📕 Índice</b></summary>
 
 - 💡 [O que é o RAGFlow?](#-o-que-é-o-ragflow)
 - 🎮 [Demo](#-demo)
@@ -75,12 +80,11 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Últimas Atualizações
 
-- 05-02-2025 Atualiza a lista de modelos de 'SILICONFLOW' e adiciona suporte para Deepseek-R1/DeepSeek-V3.
-- 26-01-2025 Otimize a extração e aplicação de gráficos de conhecimento e forneça uma variedade de opções de configuração.
-- 18-12-2024 Atualiza o modelo de Análise de Layout de Documentos no Deepdoc.
-- 04-12-2024 Adiciona suporte para pontuação de pagerank na base de conhecimento.
-- 22-11-2024 Adiciona mais variáveis para o Agente.
-- 01-11-2024 Adiciona extração de palavras-chave e geração de perguntas relacionadas aos blocos analisados para melhorar a precisão da recuperação.
+- 23-05-2025 Adicione o componente executor de código Python/JS ao Agente.
+- 05-05-2025 Suporte a consultas entre idiomas.
+- 19-03-2025 Suporta o uso de um modelo multi-modal para entender imagens dentro de arquivos PDF ou DOCX.
+- 28-02-2025 combinado com a pesquisa na Internet (T AVI LY), suporta pesquisas profundas para qualquer LLM.
+- 18-12-2024 Atualiza o modelo de Análise de Layout de Documentos no DeepDoc.
 - 22-08-2024 Suporta conversão de texto para comandos SQL via RAG.
 
 ## 🎉 Fique Ligado
@@ -133,7 +137,10 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 - RAM >= 16 GB
 - Disco >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > Se você não instalou o Docker na sua máquina local (Windows, Mac ou Linux), veja [Instalar Docker Engine](https://docs.docker.com/engine/install/).
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): Necessário apenas se você pretende usar o recurso de executor de código (sandbox) do RAGFlow.
+
+> [!TIP]
+> Se você não instalou o Docker na sua máquina local (Windows, Mac ou Linux), veja [Instalar Docker Engine](https://docs.docker.com/engine/install/).
 
 ### 🚀 Iniciar o servidor
 
@@ -166,19 +173,27 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
 3.  Inicie o servidor usando as imagens Docker pré-compiladas:
 
-    > O comando abaixo baixa a edição `v0.16.0-slim` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.16.0-slim`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor. Por exemplo: defina `RAGFLOW_IMAGE=infiniflow/ragflow:v0.16.0` para a edição completa `v0.16.0`.
+> [!CAUTION]
+> Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
+> Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
+
+    > O comando abaixo baixa a edição `v0.19.1-slim` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.19.1-slim`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor. Por exemplo: defina `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` para a edição completa `v0.19.1`.
 
     ```bash
-    $ cd ragflow
-    $ docker compose -f docker/docker-compose.yml up -d
+    $ cd ragflow/docker
+    # Use CPU for embedding and DeepDoc tasks:
+    $ docker compose -f docker-compose.yml up -d
+
+    # To use GPU to accelerate embedding and DeepDoc tasks:
+    # docker compose -f docker-compose-gpu.yml up -d
     ```
 
     | Tag da imagem RAGFlow | Tamanho da imagem (GB) | Possui modelos de incorporação? | Estável?                 |
     | --------------------- | ---------------------- | ------------------------------- | ------------------------ |
-    | v0.16.0               | ~9                     | :heavy_check_mark:              | Lançamento estável       |
-    | v0.16.0-slim          | ~2                     | ❌                              | Lançamento estável       |
+    | v0.19.1               | ~9                     | :heavy_check_mark:              | Lançamento estável       |
+    | v0.19.1-slim          | ~2                     | ❌                              | Lançamento estável       |
     | nightly               | ~9                     | :heavy_check_mark:              | _Instável_ build noturno |
-    | nightly-slim          | ~2                     | ❌                              | _Instável_ build noturno |
+    | nightly-slim          | ~2                     | ❌                               | _Instável_ build noturno |
 
 4.  Verifique o status do servidor após tê-lo iniciado:
 
@@ -196,9 +211,6 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
      /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
      * Rodando em todos os endereços (0.0.0.0)
-     * Rodando em http://127.0.0.1:9380
-     * Rodando em http://x.x.x.x:9380
-     INFO:werkzeug:Pressione CTRL+C para sair
     ```
 
     > Se você pular essa etapa de confirmação e acessar diretamente o RAGFlow, seu navegador pode exibir um erro `network anormal`, pois, nesse momento, seu RAGFlow pode não estar totalmente inicializado.
@@ -228,7 +240,7 @@ Para atualizar a porta HTTP de serviço padrão (80), vá até [docker-compose.y
 Atualizações nas configurações acima exigem um reinício de todos os contêineres para que tenham efeito:
 
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ### Mudar o mecanismo de documentos de Elasticsearch para Infinity
@@ -240,13 +252,13 @@ O RAGFlow usa o Elasticsearch por padrão para armazenar texto completo e vetore
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
-
+   Note: `-v` irá deletar os volumes do contêiner, e os dados existentes serão apagados.
 2. Defina `DOC_ENGINE` no **docker/.env** para `infinity`.
 
 3. Inicie os contêineres:
 
    ```bash
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
 
 > [!ATENÇÃO]
@@ -259,7 +271,7 @@ Esta imagem tem cerca de 2 GB de tamanho e depende de serviços externos de LLM
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 Criar uma imagem Docker incluindo modelos de incorporação
@@ -269,7 +281,7 @@ Esta imagem tem cerca de 9 GB de tamanho. Como inclui modelos de incorporação,
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 Lançar o serviço a partir do código-fonte para desenvolvimento
@@ -277,7 +289,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 1. Instale o `uv`, ou pule esta etapa se ele já estiver instalado:
 
    ```bash
-   pipx install uv
+   pipx install uv pre-commit
    ```
 
 2. Clone o código-fonte e instale as dependências Python:
@@ -286,6 +298,8 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
    uv sync --python 3.10 --all-extras # instala os módulos Python dependentes do RAGFlow
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. Inicie os serviços dependentes (MinIO, Elasticsearch, Redis e MySQL) usando Docker Compose:
@@ -297,7 +311,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    Adicione a seguinte linha ao arquivo `/etc/hosts` para resolver todos os hosts especificados em **docker/.env** para `127.0.0.1`:
 
    ```
-   127.0.0.1       es01 infinity mysql minio redis
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
 
 4. Se não conseguir acessar o HuggingFace, defina a variável de ambiente `HF_ENDPOINT` para usar um site espelho:
@@ -306,7 +320,16 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. Lance o serviço de back-end:
+5. Se o seu sistema operacional não tiver jemalloc, instale-o da seguinte maneira:
+
+    ```bash
+    # ubuntu
+    sudo apt-get install libjemalloc-dev
+    # centos
+    sudo yum instalar jemalloc
+    ```
+
+6. Lance o serviço de back-end:
 
    ```bash
    source .venv/bin/activate
@@ -314,14 +337,14 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
    bash docker/launch_backend_service.sh
    ```
 
-6. Instale as dependências do front-end:
+7. Instale as dependências do front-end:
 
    ```bash
    cd web
    npm install
    ```
 
-7. Lance o serviço de front-end:
+8. Lance o serviço de front-end:
 
    ```bash
    npm run dev
@@ -331,12 +354,22 @@ docker build -f Dockerfile -t infiniflow/ragflow:nightly .
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+9. Pare os serviços de front-end e back-end do RAGFlow após a conclusão do desenvolvimento:
+
+    ```bash
+    pkill -f "ragflow_server.py|task_executor.py"
+    ```
+
+
 ## 📚 Documentação
 
-- [Início rápido](https://ragflow.io/docs/dev/)
-- [Guia do usuário](https://ragflow.io/docs/dev/category/guides)
-- [Referências](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [Quickstart](https://ragflow.io/docs/dev/)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
+- [References](https://ragflow.io/docs/dev/category/references)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 Roadmap
 
@@ -344,11 +377,11 @@ Veja o [RAGFlow Roadmap 2025](https://github.com/infiniflow/ragflow/issues/4214)
 
 ## 🏄 Comunidade
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/NjYzJD3GM3)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Contribuindo
 
 O RAGFlow prospera por meio da colaboração de código aberto. Com esse espírito, abraçamos contribuições diversas da comunidade.
-Se você deseja fazer parte, primeiro revise nossas [Diretrizes de Contribuição](./CONTRIBUTING.md).
+Se você deseja fazer parte, primeiro revise nossas [Diretrizes de Contribuição](https://ragflow.io/docs/dev/contributing).

README_tzh.md

@@ -5,12 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DFE0E5"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DBEDFA"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -21,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.16.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.16.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -29,16 +30,44 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
+
+<details open>
+<summary><b>📕 目錄</b></summary>
+
+- 💡 [RAGFlow 是什麼？](#-RAGFlow-是什麼)
+- 🎮 [Demo-試用](#-demo-試用)
+- 📌 [近期更新](#-近期更新)
+- 🌟 [主要功能](#-主要功能)
+- 🔎 [系統架構](#-系統架構)
+- 🎬 [快速開始](#-快速開始)
+- 🔧 [系統配置](#-系統配置)
+- 🔨 [以原始碼啟動服務](#-以原始碼啟動服務)
+- 📚 [技術文檔](#-技術文檔)
+- 📜 [路線圖](#-路線圖)
+- 🏄 [貢獻指南](#-貢獻指南)
+- 🙌 [加入社區](#-加入社區)
+- 🤝 [商務合作](#-商務合作)
+
+</details>
+
 ## 💡 RAGFlow 是什麼？
 
 [RAGFlow](https://ragflow.io/) 是一款基於深度文件理解所建構的開源 RAG（Retrieval-Augmented Generation）引擎。 RAGFlow 可以為各種規模的企業及個人提供一套精簡的 RAG 工作流程，結合大語言模型（LLM）針對用戶各類不同的複雜格式數據提供可靠的問答以及有理有據的引用。
@@ -54,12 +83,11 @@
 
 ## 🔥 近期更新
 
-- 2025-02-05 更新「SILICONFLOW」的型號清單並新增 Deepseek-R1/DeepSeek-V3 的支援。
-- 2025-01-26 最佳化知識圖譜的擷取與應用，提供了多種配置選擇。
-- 2024-12-18 升級了 Deepdoc 的文檔佈局分析模型。
-- 2024-12-04 支援知識庫的 Pagerank 分數。
-- 2024-11-22 完善了 Agent 中的變數定義和使用。
-- 2024-11-01 對解析後的 chunk 加入關鍵字抽取和相關問題產生以提高回想的準確度。
+- 2025-05-23 為 Agent 新增 Python/JS 程式碼執行器元件。
+- 2025-05-05 支援跨語言查詢。
+- 2025-03-19 PDF和DOCX中的圖支持用多模態大模型去解析得到描述.
+- 2025-02-28 結合網路搜尋（Tavily），對於任意大模型實現類似 Deep Research 的推理功能.
+- 2024-12-18 升級了 DeepDoc 的文檔佈局分析模型。
 - 2024-08-22 支援用 RAG 技術實現從自然語言到 SQL 語句的轉換。
 
 ## 🎉 關注項目
@@ -112,7 +140,10 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > 如果你並沒有在本機安裝 Docker（Windows、Mac，或 Linux）, 可以參考文件 [Install Docker Engine](https://docs.docker.com/engine/install/) 自行安裝。
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): 僅在您打算使用 RAGFlow 的代碼執行器（沙箱）功能時才需要安裝。
+
+> [!TIP]
+> 如果你並沒有在本機安裝 Docker（Windows、Mac，或 Linux）, 可以參考文件 [Install Docker Engine](https://docs.docker.com/engine/install/) 自行安裝。
 
 ### 🚀 啟動伺服器
 
@@ -145,19 +176,27 @@
 
 3. 進入 **docker** 資料夾，利用事先編譯好的 Docker 映像啟動伺服器：
 
-   > 執行以下指令會自動下載 RAGFlow slim Docker 映像 `v0.16.0-slim`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.16.0-slim` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。例如，你可以透過設定 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.16.0` 來下載 RAGFlow 鏡像的 `v0.16.0` 完整發行版。
+> [!CAUTION]
+> 所有 Docker 映像檔都是為 x86 平台建置的。目前，我們不提供 ARM64 平台的 Docker 映像檔。
+> 如果您使用的是 ARM64 平台，請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
+
+   > 執行以下指令會自動下載 RAGFlow slim Docker 映像 `v0.19.1-slim`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.19.1-slim` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。例如，你可以透過設定 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` 來下載 RAGFlow 鏡像的 `v0.19.1` 完整發行版。
 
    ```bash
-   $ cd ragflow
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
    ```
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.16.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.16.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 
    > [!TIP]
    > 如果你遇到 Docker 映像檔拉不下來的問題，可以在 **docker/.env** 檔案內根據變數 `RAGFLOW_IMAGE` 的註解提示選擇華為雲或阿里雲的對應映像。
@@ -181,9 +220,6 @@
     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
 
    > 如果您跳過這一步驟系統確認步驟就登入 RAGFlow，你的瀏覽器有可能會提示 `network anormal` 或 `網路異常`，因為 RAGFlow 可能並未完全啟動成功。
@@ -200,7 +236,7 @@
 
 系統配置涉及以下三份文件：
 
-- [.env](./docker/.env)：存放一些基本的系統環境變量，例如 `SVR_HTTP_PORT`、`MYSQL_PASSWORD`、`MINIO_PASSWORD` 等。
+- [.env](./docker/.env)：存放一些系統環境變量，例如 `SVR_HTTP_PORT`、`MYSQL_PASSWORD`、`MINIO_PASSWORD` 等。
 - [service_conf.yaml.template](./docker/service_conf.yaml.template)：設定各類別後台服務。
 - [docker-compose.yml](./docker/docker-compose.yml): 系統依賴該檔案完成啟動。
 
@@ -215,7 +251,7 @@
 > 所有系統配置都需要透過系統重新啟動生效：
 >
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ###把文檔引擎從 Elasticsearch 切換成為 Infinity
@@ -227,13 +263,14 @@ RAGFlow 預設使用 Elasticsearch 儲存文字和向量資料. 如果要切換
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+   Note: `-v` 將會刪除 docker 容器的 volumes，已有的資料會被清空。
 
 2. 設定 **docker/.env** 目錄中的 `DOC_ENGINE` 為 `infinity`.
 
 3. 啟動容器:
 
    ```bash
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
 
 > [!WARNING]
@@ -246,7 +283,7 @@ RAGFlow 預設使用 Elasticsearch 儲存文字和向量資料. 如果要切換
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 原始碼編譯 Docker 映像（包含 embedding 模型）
@@ -256,7 +293,7 @@ docker build --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t in
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 以原始碼啟動服務
@@ -264,8 +301,8 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
 1. 安裝 uv。如已安裝，可跳過此步驟：
 
    ```bash
-   pipx install uv
-   export UV_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
+   pipx install uv pre-commit
+   export UV_INDEX=https://mirrors.aliyun.com/pypi/simple
    ```
 
 2. 下載原始碼並安裝 Python 依賴：
@@ -274,6 +311,8 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
    uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. 透過 Docker Compose 啟動依賴的服務（MinIO, Elasticsearch, Redis, and MySQL）：
@@ -285,7 +324,7 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
    在 `/etc/hosts` 中加入以下程式碼，將 **conf/service_conf.yaml** 檔案中的所有 host 位址都解析為 `127.0.0.1`：
 
    ```
-   127.0.0.1 es01 infinity mysql minio redis
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
 
 4. 如果無法存取 HuggingFace，可以把環境變數 `HF_ENDPOINT` 設為對應的鏡像網站：
@@ -294,24 +333,34 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5.啟動後端服務：
-『`bash
-source .venv/bin/activate
-export PYTHONPATH=$(pwd)
-bash docker/launch_backend_service.sh
+5. 如果你的操作系统没有 jemalloc，请按照如下方式安装：
 
-```
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
 
-6. 安裝前端依賴：
-『`bash
-cd web
-npm install
-```
+6. 啟動後端服務：
 
-7. 啟動前端服務：
-   『`bash
+   ```bash
+   source .venv/bin/activate
+   export PYTHONPATH=$(pwd)
+   bash docker/launch_backend_service.sh
+   ```
+
+7. 安裝前端依賴：
+
+   ```bash
+   cd web
+   npm install
+   ```
+
+8. 啟動前端服務：
+
+   ```bash
    npm run dev
-
    ```
 
    以下界面說明系統已成功啟動：_
@@ -319,12 +368,22 @@ npm install
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
    ```
 
+9. 開發完成後停止 RAGFlow 前端和後端服務：
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
 ## 📚 技術文檔
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/guides)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
 - [References](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 路線圖
 
@@ -332,13 +391,13 @@ npm install
 
 ## 🏄 開源社群
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/zd4qPW6t)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 貢獻指南
 
-RAGFlow 只有透過開源協作才能蓬勃發展。秉持這項精神,我們歡迎來自社區的各種貢獻。如果您有意參與其中,請查閱我們的 [貢獻者指南](./CONTRIBUTING.md) 。
+RAGFlow 只有透過開源協作才能蓬勃發展。秉持這項精神,我們歡迎來自社區的各種貢獻。如果您有意參與其中,請查閱我們的 [貢獻者指南](https://ragflow.io/docs/dev/contributing) 。
 
 ## 🤝 商務合作
 

README_zh.md

@@ -5,13 +5,13 @@
 </div>
 
 <p align="center">
-  <a href="./README.md">English</a> |
-  <a href="./README_zh.md">简体中文</a> |
-  <a href="./README_tzh.md">繁体中文</a> |
-  <a href="./README_ja.md">日本語</a> |
-  <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a> |
-  <a href="/README_pt_br.md">Português (Brasil)</a>
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-DFE0E5"></a>
+  <a href="./README_zh.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-DBEDFA"></a>
+  <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
+  <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
+  <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
+  <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
 </p>
 
 <p align="center">
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.16.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.16.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,16 +30,44 @@
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
         <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
     </a>
+    <a href="https://deepwiki.com/infiniflow/ragflow">
+        <img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg">
+    </a>
 </p>
 
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
+#
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
+
+<details open>
+<summary><b>📕 目录</b></summary>
+
+- 💡 [RAGFlow 是什么？](#-RAGFlow-是什么)
+- 🎮 [Demo](#-demo)
+- 📌 [近期更新](#-近期更新)
+- 🌟 [主要功能](#-主要功能)
+- 🔎 [系统架构](#-系统架构)
+- 🎬 [快速开始](#-快速开始)
+- 🔧 [系统配置](#-系统配置)
+- 🔨 [以源代码启动服务](#-以源代码启动服务)
+- 📚 [技术文档](#-技术文档)
+- 📜 [路线图](#-路线图)
+- 🏄 [贡献指南](#-贡献指南)
+- 🙌 [加入社区](#-加入社区)
+- 🤝 [商务合作](#-商务合作)
+
+</details>
+
 ## 💡 RAGFlow 是什么？
 
 [RAGFlow](https://ragflow.io/) 是一款基于深度文档理解构建的开源 RAG（Retrieval-Augmented Generation）引擎。RAGFlow 可以为各种规模的企业及个人提供一套精简的 RAG 工作流程，结合大语言模型（LLM）针对用户各类不同的复杂格式数据提供可靠的问答以及有理有据的引用。
@@ -55,12 +83,11 @@
 
 ## 🔥 近期更新
 
-- 2025-02-05 更新硅基流动的模型列表，增加了对 Deepseek-R1/DeepSeek-V3 的支持。
-- 2025-01-26 优化知识图谱的提取和应用，提供了多种配置选择。
-- 2024-12-18 升级了 Deepdoc 的文档布局分析模型。
-- 2024-12-04 支持知识库的 Pagerank 分数。
-- 2024-11-22 完善了 Agent 中的变量定义和使用。
-- 2024-11-01 对解析后的 chunk 加入关键词抽取和相关问题生成以提高召回的准确度。
+- 2025-05-23 Agent 新增 Python/JS 代码执行器组件。
+- 2025-05-05 支持跨语言查询。
+- 2025-03-19 PDF 和 DOCX 中的图支持用多模态大模型去解析得到描述.
+- 2025-02-28 结合互联网搜索（Tavily），对于任意大模型实现类似 Deep Research 的推理功能.
+- 2024-12-18 升级了 DeepDoc 的文档布局分析模型。
 - 2024-08-22 支持用 RAG 技术实现从自然语言到 SQL 语句的转换。
 
 ## 🎉 关注项目
@@ -113,7 +140,10 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > 如果你并没有在本机安装 Docker（Windows、Mac，或者 Linux）, 可以参考文档 [Install Docker Engine](https://docs.docker.com/engine/install/) 自行安装。
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): 仅在你打算使用 RAGFlow 的代码执行器（沙箱）功能时才需要安装。
+
+> [!TIP]
+> 如果你并没有在本机安装 Docker（Windows、Mac，或者 Linux）, 可以参考文档 [Install Docker Engine](https://docs.docker.com/engine/install/) 自行安装。
 
 ### 🚀 启动服务器
 
@@ -146,19 +176,27 @@
 
 3. 进入 **docker** 文件夹，利用提前编译好的 Docker 镜像启动服务器：
 
-   > 运行以下命令会自动下载 RAGFlow slim Docker 镜像 `v0.16.0-slim`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.16.0-slim` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。比如，你可以通过设置 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.16.0` 来下载 RAGFlow 镜像的 `v0.16.0` 完整发行版。
+> [!CAUTION]
+> 请注意，目前官方提供的所有 Docker 镜像均基于 x86 架构构建，并不提供基于 ARM64 的 Docker 镜像。
+> 如果你的操作系统是 ARM64 架构，请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
+
+   > 运行以下命令会自动下载 RAGFlow slim Docker 镜像 `v0.19.1-slim`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.19.1-slim` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。比如，你可以通过设置 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` 来下载 RAGFlow 镜像的 `v0.19.1` 完整发行版。
 
    ```bash
-   $ cd ragflow
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
    ```
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.16.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.16.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.1-slim      | &approx;2       | ❌                    | Stable release           |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 
    > [!TIP]
    > 如果你遇到 Docker 镜像拉不下来的问题，可以在 **docker/.env** 文件内根据变量 `RAGFLOW_IMAGE` 的注释提示选择华为云或者阿里云的相应镜像。
@@ -182,12 +220,9 @@
     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
 
-   > 如果您跳过这一步系统确认步骤就登录 RAGFlow，你的浏览器有可能会提示 `network anormal` 或 `网络异常`，因为 RAGFlow 可能并未完全启动成功。
+   > 如果您在没有看到上面的提示信息出来之前，就尝试登录 RAGFlow，你的浏览器有可能会提示 `network anormal` 或 `网络异常`。
 
 5. 在你的浏览器中输入你的服务器对应的 IP 地址并登录 RAGFlow。
    > 上面这个例子中，您只需输入 http://IP_OF_YOUR_MACHINE 即可：未改动过配置则无需输入端口（默认的 HTTP 服务端口 80）。
@@ -216,7 +251,7 @@
 > 所有系统配置都需要通过系统重启生效：
 >
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ### 把文档引擎从 Elasticsearch 切换成为 Infinity
@@ -228,13 +263,14 @@ RAGFlow 默认使用 Elasticsearch 存储文本和向量数据. 如果要切换
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+   Note: `-v` 将会删除 docker 容器的 volumes，已有的数据会被清空。
 
 2. 设置 **docker/.env** 目录中的 `DOC_ENGINE` 为 `infinity`.
 
 3. 启动容器:
 
    ```bash
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
 
 > [!WARNING]
@@ -247,7 +283,7 @@ RAGFlow 默认使用 Elasticsearch 存储文本和向量数据. 如果要切换
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 源码编译 Docker 镜像（包含 embedding 模型）
@@ -257,7 +293,7 @@ docker build --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t in
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 以源代码启动服务
@@ -265,8 +301,8 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
 1. 安装 uv。如已经安装，可跳过本步骤：
 
    ```bash
-   pipx install uv
-   export UV_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
+   pipx install uv pre-commit
+   export UV_INDEX=https://mirrors.aliyun.com/pypi/simple
    ```
 
 2. 下载源代码并安装 Python 依赖：
@@ -275,6 +311,8 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
    uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. 通过 Docker Compose 启动依赖的服务（MinIO, Elasticsearch, Redis, and MySQL）：
@@ -283,19 +321,27 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
    docker compose -f docker/docker-compose-base.yml up -d
    ```
 
-   在 `/etc/hosts` 中添加以下代码，将 **conf/service_conf.yaml** 文件中的所有 host 地址都解析为 `127.0.0.1`：
+   在 `/etc/hosts` 中添加以下代码，目的是将 **conf/service_conf.yaml** 文件中的所有 host 地址都解析为 `127.0.0.1`：
 
    ```
-   127.0.0.1       es01 infinity mysql minio redis
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. 如果无法访问 HuggingFace，可以把环境变量 `HF_ENDPOINT` 设成相应的镜像站点：
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. 启动后端服务：
+5. 如果你的操作系统没有 jemalloc，请按照如下方式安装：
+
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+
+6. 启动后端服务：
 
    ```bash
    source .venv/bin/activate
@@ -303,12 +349,14 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
    bash docker/launch_backend_service.sh
    ```
 
-6. 安装前端依赖：
+7. 安装前端依赖：
+
    ```bash
    cd web
    npm install
    ```
-7. 启动前端服务：
+
+8. 启动前端服务：
 
    ```bash
    npm run dev
@@ -318,12 +366,22 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+9. 开发完成后停止 RAGFlow 前端和后端服务：
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
 ## 📚 技术文档
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/guides)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
 - [References](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 路线图
 
@@ -331,13 +389,13 @@ docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:night
 
 ## 🏄 开源社区
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/zd4qPW6t)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 贡献指南
 
-RAGFlow 只有通过开源协作才能蓬勃发展。秉持这一精神,我们欢迎来自社区的各种贡献。如果您有意参与其中,请查阅我们的 [贡献者指南](./CONTRIBUTING.md) 。
+RAGFlow 只有通过开源协作才能蓬勃发展。秉持这一精神,我们欢迎来自社区的各种贡献。如果您有意参与其中,请查阅我们的 [贡献者指南](https://ragflow.io/docs/dev/contributing) 。
 
 ## 🤝 商务合作
 

agent/canvas.py

@@ -15,17 +15,15 @@
 #
 import logging
 import json
-from abc import ABC
 from copy import deepcopy
 from functools import partial
-
 import pandas as pd
 
 from agent.component import component_class
 from agent.component.base import ComponentBase
 
 
-class Canvas(ABC):
+class Canvas:
     """
     dsl = {
         "components": {
@@ -162,13 +160,17 @@ class Canvas(ABC):
             self.components[k]["obj"].reset()
         self._embed_id = ""
 
-    def get_compnent_name(self, cid):
+    def get_component_name(self, cid):
         for n in self.dsl["graph"]["nodes"]:
             if cid == n["id"]:
                 return n["data"]["name"]
         return ""
 
-    def run(self, **kwargs):
+    def run(self, running_hint_text = "is running...🕞", **kwargs):
+        if not running_hint_text or not isinstance(running_hint_text, str):
+            running_hint_text = "is running...🕞"
+        bypass_begin = bool(kwargs.get("bypass_begin", False))
+
         if self.answer:
             cpn_id = self.answer[0]
             self.answer.pop(0)
@@ -187,6 +189,12 @@ class Canvas(ABC):
         if not self.path:
             self.components["begin"]["obj"].run(self.history, **kwargs)
             self.path.append(["begin"])
+            if bypass_begin:
+                cpn = self.get_component("begin")
+                downstream = cpn["downstream"]
+                self.path.append(downstream)
+
+
 
         self.path.append([])
 
@@ -210,7 +218,7 @@ class Canvas(ABC):
                             if c not in waiting:
                                 waiting.append(c)
                             continue
-                    yield "*'{}'* is running...🕞".format(self.get_compnent_name(c))
+                    yield "*'{}'* {}".format(self.get_component_name(c), running_hint_text)
 
                     if cpn.component_name.lower() == "iteration":
                         st_cpn = cpn.get_start()
@@ -236,7 +244,7 @@ class Canvas(ABC):
             pid = self.components[cid]["parent_id"]
             o, _ = self.components[cid]["obj"].output(allow_partial=False)
             oo, _ = self.components[pid]["obj"].output(allow_partial=False)
-            self.components[pid]["obj"].set(pd.concat([oo, o], ignore_index=True))
+            self.components[pid]["obj"].set_output(pd.concat([oo, o], ignore_index=True).dropna())
             downstream = [pid]
 
         for m in prepare2run(downstream):
@@ -253,20 +261,20 @@ class Canvas(ABC):
             if loop:
                 raise OverflowError(f"Too much loops: {loop}")
 
+            downstream = []
             if cpn["obj"].component_name.lower() in ["switch", "categorize", "relevant"]:
                 switch_out = cpn["obj"].output()[1].iloc[0, 0]
                 assert switch_out in self.components, \
                     "{}'s output: {} not valid.".format(cpn_id, switch_out)
-                for m in prepare2run([switch_out]):
-                    yield {"content": m, "running_status": True}
-                continue
+                downstream = [switch_out]
+            else:
+                downstream = cpn["downstream"]
 
-            downstream = cpn["downstream"]
             if not downstream and cpn.get("parent_id"):
                 pid = cpn["parent_id"]
                 _, o = cpn["obj"].output(allow_partial=False)
                 _, oo = self.components[pid]["obj"].output(allow_partial=False)
-                self.components[pid]["obj"].set_output(pd.concat([oo.dropna(axis=1), o.dropna(axis=1)], ignore_index=True))
+                self.components[pid]["obj"].set_output(pd.concat([oo.dropna(axis=1), o.dropna(axis=1)], ignore_index=True).dropna())
                 downstream = [pid]
 
             for m in prepare2run(downstream):
@@ -303,6 +311,8 @@ class Canvas(ABC):
 
     def get_history(self, window_size):
         convs = []
+        if window_size <= 0:
+            return convs
         for role, obj in self.history[window_size * -1:]:
             if isinstance(obj, list) and obj and all([isinstance(o, dict) for o in obj]):
                 convs.append({"role": role, "content": '\n'.join([str(s.get("content", "")) for s in obj])})
@@ -363,4 +373,7 @@ class Canvas(ABC):
         return self.components["begin"]["obj"]._param.query
 
     def get_component_input_elements(self, cpnnm):
-        return self.components[cpnnm]["obj"].get_input_elements()
+        return self.components[cpnnm]["obj"].get_input_elements()
+    
+    def set_component_infor(self, cpn_id, infor):
+        self.components[cpn_id]["obj"].set_infor(infor)

agent/component/__init__.py

@@ -50,6 +50,7 @@ from .template import Template, TemplateParam
 from .email import Email, EmailParam
 from .iteration import Iteration, IterationParam
 from .iterationitem import IterationItem, IterationItemParam
+from .code import Code, CodeParam
 
 
 def component_class(class_name):
@@ -129,5 +130,7 @@ __all__ = [
     "TemplateParam",
     "Email",
     "EmailParam",
+    "Code",
+    "CodeParam",
     "component_class"
 ]

agent/component/answer.py

@@ -64,14 +64,17 @@ class Answer(ComponentBase, ABC):
             for ii, row in stream.iterrows():
                 answer += row.to_dict()["content"]
                 yield {"content": answer}
-        else:
+        elif stream is not None:
             for st in stream():
                 res = st
                 yield st
-        if self._param.post_answers:
+        if self._param.post_answers and res:
             res["content"] += random.choice(self._param.post_answers)
             yield res
 
+        if res is None:
+            res = {"content": ""}
+
         self.set_output(res)
 
     def set_exception(self, e):

agent/component/baidu.py

@@ -17,6 +17,7 @@ import logging
 from abc import ABC
 import pandas as pd
 import requests
+from bs4 import BeautifulSoup
 import re
 from agent.component.base import ComponentBase, ComponentParamBase
 
@@ -44,17 +45,28 @@ class Baidu(ComponentBase, ABC):
             return Baidu.be_output("")
 
         try:
-            url = 'http://www.baidu.com/s?wd=' + ans + '&rn=' + str(self._param.top_n)
+            url = 'https://www.baidu.com/s?wd=' + ans + '&rn=' + str(self._param.top_n)
             headers = {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'}
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+                'Connection': 'keep-alive',
+            }
             response = requests.get(url=url, headers=headers)
-
-            url_res = re.findall(r"'url': \\\"(.*?)\\\"}", response.text)
-            title_res = re.findall(r"'title': \\\"(.*?)\\\",\\n", response.text)
-            body_res = re.findall(r"\"contentText\":\"(.*?)\"", response.text)
-            baidu_res = [{"content": re.sub('<em>|</em>', '', '<a href="' + url + '">' + title + '</a>    ' + body)} for
-                         url, title, body in zip(url_res, title_res, body_res)]
-            del body_res, url_res, title_res
+            # check if request success
+            if response.status_code == 200:
+                soup = BeautifulSoup(response.text, 'html.parser')
+                url_res = []
+                title_res = []
+                body_res = []
+                for item in soup.select('.result.c-container'):
+                    # extract title
+                    title_res.append(item.select_one('h3 a').get_text(strip=True))
+                    url_res.append(item.select_one('h3 a')['href'])
+                    body_res.append(item.select_one('.c-abstract').get_text(strip=True) if item.select_one('.c-abstract') else '')
+                baidu_res = [{"content": re.sub('<em>|</em>', '', '<a href="' + url + '">' + title + '</a>    ' + body)} for
+                             url, title, body in zip(url_res, title_res, body_res)]
+                del body_res, url_res, title_res
         except Exception as e:
             return Baidu.be_output("**ERROR**: " + str(e))
 

agent/component/base.py

@@ -13,13 +13,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-from abc import ABC
 import builtins
 import json
-import os
 import logging
+import os
+from abc import ABC
 from functools import partial
-from typing import Tuple, Union
+from typing import Any, Tuple, Union
 
 import pandas as pd
 
@@ -34,6 +34,7 @@ _IS_RAW_CONF = "_is_raw_conf"
 class ComponentParamBase(ABC):
     def __init__(self):
         self.output_var_name = "output"
+        self.infor_var_name = "infor"
         self.message_history_window_size = 22
         self.query = []
         self.inputs = []
@@ -109,15 +110,11 @@ class ComponentParamBase(ABC):
         update_from_raw_conf = conf.get(_IS_RAW_CONF, True)
         if update_from_raw_conf:
             deprecated_params_set = self._get_or_init_deprecated_params_set()
-            feeded_deprecated_params_set = (
-                self._get_or_init_feeded_deprecated_params_set()
-            )
+            feeded_deprecated_params_set = self._get_or_init_feeded_deprecated_params_set()
             user_feeded_params_set = self._get_or_init_user_feeded_params_set()
             setattr(self, _IS_RAW_CONF, False)
         else:
-            feeded_deprecated_params_set = (
-                self._get_or_init_feeded_deprecated_params_set(conf)
-            )
+            feeded_deprecated_params_set = self._get_or_init_feeded_deprecated_params_set(conf)
             user_feeded_params_set = self._get_or_init_user_feeded_params_set(conf)
 
         def _recursive_update_param(param, config, depth, prefix):
@@ -153,15 +150,11 @@ class ComponentParamBase(ABC):
 
                 else:
                     # recursive set obj attr
-                    sub_params = _recursive_update_param(
-                        attr, config_value, depth + 1, prefix=f"{prefix}{config_key}."
-                    )
+                    sub_params = _recursive_update_param(attr, config_value, depth + 1, prefix=f"{prefix}{config_key}.")
                     setattr(param, config_key, sub_params)
 
             if not allow_redundant and redundant_attrs:
-                raise ValueError(
-                    f"cpn `{getattr(self, '_name', type(self))}` has redundant parameters: `{[redundant_attrs]}`"
-                )
+                raise ValueError(f"cpn `{getattr(self, '_name', type(self))}` has redundant parameters: `{[redundant_attrs]}`")
 
             return param
 
@@ -192,9 +185,7 @@ class ComponentParamBase(ABC):
         param_validation_path_prefix = home_dir + "/param_validation/"
 
         param_name = type(self).__name__
-        param_validation_path = "/".join(
-            [param_validation_path_prefix, param_name + ".json"]
-        )
+        param_validation_path = "/".join([param_validation_path_prefix, param_name + ".json"])
 
         validation_json = None
 
@@ -227,11 +218,7 @@ class ComponentParamBase(ABC):
                         break
 
                 if not value_legal:
-                    raise ValueError(
-                        "Plase check runtime conf, {} = {} does not match user-parameter restriction".format(
-                            variable, value
-                        )
-                    )
+                    raise ValueError("Plase check runtime conf, {} = {} does not match user-parameter restriction".format(variable, value))
 
             elif variable in validation_json:
                 self._validate_param(attr, validation_json)
@@ -239,94 +226,63 @@ class ComponentParamBase(ABC):
     @staticmethod
     def check_string(param, descr):
         if type(param).__name__ not in ["str"]:
-            raise ValueError(
-                descr + " {} not supported, should be string type".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be string type".format(param))
 
     @staticmethod
     def check_empty(param, descr):
         if not param:
-            raise ValueError(
-                descr + " does not support empty value."
-            )
+            raise ValueError(descr + " does not support empty value.")
 
     @staticmethod
     def check_positive_integer(param, descr):
         if type(param).__name__ not in ["int", "long"] or param <= 0:
-            raise ValueError(
-                descr + " {} not supported, should be positive integer".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be positive integer".format(param))
 
     @staticmethod
     def check_positive_number(param, descr):
         if type(param).__name__ not in ["float", "int", "long"] or param <= 0:
-            raise ValueError(
-                descr + " {} not supported, should be positive numeric".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be positive numeric".format(param))
 
     @staticmethod
     def check_nonnegative_number(param, descr):
         if type(param).__name__ not in ["float", "int", "long"] or param < 0:
-            raise ValueError(
-                descr
-                + " {} not supported, should be non-negative numeric".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be non-negative numeric".format(param))
 
     @staticmethod
     def check_decimal_float(param, descr):
         if type(param).__name__ not in ["float", "int"] or param < 0 or param > 1:
-            raise ValueError(
-                descr
-                + " {} not supported, should be a float number in range [0, 1]".format(
-                    param
-                )
-            )
+            raise ValueError(descr + " {} not supported, should be a float number in range [0, 1]".format(param))
 
     @staticmethod
     def check_boolean(param, descr):
         if type(param).__name__ != "bool":
-            raise ValueError(
-                descr + " {} not supported, should be bool type".format(param)
-            )
+            raise ValueError(descr + " {} not supported, should be bool type".format(param))
 
     @staticmethod
     def check_open_unit_interval(param, descr):
         if type(param).__name__ not in ["float"] or param <= 0 or param >= 1:
-            raise ValueError(
-                descr + " should be a numeric number between 0 and 1 exclusively"
-            )
+            raise ValueError(descr + " should be a numeric number between 0 and 1 exclusively")
 
     @staticmethod
     def check_valid_value(param, descr, valid_values):
         if param not in valid_values:
-            raise ValueError(
-                descr
-                + " {} is not supported, it should be in {}".format(param, valid_values)
-            )
+            raise ValueError(descr + " {} is not supported, it should be in {}".format(param, valid_values))
 
     @staticmethod
     def check_defined_type(param, descr, types):
         if type(param).__name__ not in types:
-            raise ValueError(
-                descr + " {} not supported, should be one of {}".format(param, types)
-            )
+            raise ValueError(descr + " {} not supported, should be one of {}".format(param, types))
 
     @staticmethod
     def check_and_change_lower(param, valid_list, descr=""):
         if type(param).__name__ != "str":
-            raise ValueError(
-                descr
-                + " {} not supported, should be one of {}".format(param, valid_list)
-            )
+            raise ValueError(descr + " {} not supported, should be one of {}".format(param, valid_list))
 
         lower_param = param.lower()
         if lower_param in valid_list:
             return lower_param
         else:
-            raise ValueError(
-                descr
-                + " {} not supported, should be one of {}".format(param, valid_list)
-            )
+            raise ValueError(descr + " {} not supported, should be one of {}".format(param, valid_list))
 
     @staticmethod
     def _greater_equal_than(value, limit):
@@ -340,11 +296,7 @@ class ComponentParamBase(ABC):
     def _range(value, ranges):
         in_range = False
         for left_limit, right_limit in ranges:
-            if (
-                    left_limit - settings.FLOAT_ZERO
-                    <= value
-                    <= right_limit + settings.FLOAT_ZERO
-            ):
+            if left_limit - settings.FLOAT_ZERO <= value <= right_limit + settings.FLOAT_ZERO:
                 in_range = True
                 break
 
@@ -360,16 +312,11 @@ class ComponentParamBase(ABC):
 
     def _warn_deprecated_param(self, param_name, descr):
         if self._deprecated_params_set.get(param_name):
-            logging.warning(
-                f"{descr} {param_name} is deprecated and ignored in this version."
-            )
+            logging.warning(f"{descr} {param_name} is deprecated and ignored in this version.")
 
     def _warn_to_deprecate_param(self, param_name, descr, new_param):
         if self._deprecated_params_set.get(param_name):
-            logging.warning(
-                f"{descr} {param_name} will be deprecated in future release; "
-                f"please use {new_param} instead."
-            )
+            logging.warning(f"{descr} {param_name} will be deprecated in future release; please use {new_param} instead.")
             return True
         return False
 
@@ -384,33 +331,44 @@ class ComponentBase(ABC):
             "params": {}
         }
         """
+        out = getattr(self._param, self._param.output_var_name)
+        if isinstance(out, pd.DataFrame) and "chunks" in out:
+            del out["chunks"]
+            setattr(self._param, self._param.output_var_name, out)
+
         return """{{
             "component_name": "{}",
             "params": {},
             "output": {},
             "inputs": {}
-        }}""".format(self.component_name,
-                     self._param,
-                     json.dumps(json.loads(str(self._param)).get("output", {}), ensure_ascii=False),
-                     json.dumps(json.loads(str(self._param)).get("inputs", []), ensure_ascii=False)
+        }}""".format(
+            self.component_name,
+            self._param,
+            json.dumps(json.loads(str(self._param)).get("output", {}), ensure_ascii=False),
+            json.dumps(json.loads(str(self._param)).get("inputs", []), ensure_ascii=False),
         )
 
     def __init__(self, canvas, id, param: ComponentParamBase):
+        from agent.canvas import Canvas  # Local import to avoid cyclic dependency
+
+        assert isinstance(canvas, Canvas), "canvas must be an instance of Canvas"
         self._canvas = canvas
         self._id = id
         self._param = param
         self._param.check()
 
     def get_dependent_components(self):
-        cpnts = set([para["component_id"].split("@")[0] for para in self._param.query \
-                     if para.get("component_id") \
-                     and para["component_id"].lower().find("answer") < 0 \
-                     and para["component_id"].lower().find("begin") < 0])
+        cpnts = set(
+            [
+                para["component_id"].split("@")[0]
+                for para in self._param.query
+                if para.get("component_id") and para["component_id"].lower().find("answer") < 0 and para["component_id"].lower().find("begin") < 0
+            ]
+        )
         return list(cpnts)
 
     def run(self, history, **kwargs):
-        logging.debug("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False),
-                                                              json.dumps(kwargs, ensure_ascii=False)))
+        logging.debug("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False), json.dumps(kwargs, ensure_ascii=False)))
         self._param.debug_inputs = []
         try:
             res = self._run(history, **kwargs)
@@ -429,7 +387,7 @@ class ComponentBase(ABC):
         if not isinstance(o, partial):
             if not isinstance(o, pd.DataFrame):
                 if isinstance(o, list):
-                    return self._param.output_var_name, pd.DataFrame(o)
+                    return self._param.output_var_name, pd.DataFrame(o).dropna()
                 if o is None:
                     return self._param.output_var_name, pd.DataFrame()
                 return self._param.output_var_name, pd.DataFrame([{"content": str(o)}])
@@ -437,15 +395,15 @@ class ComponentBase(ABC):
 
         if allow_partial or not isinstance(o, partial):
             if not isinstance(o, partial) and not isinstance(o, pd.DataFrame):
-                return pd.DataFrame(o if isinstance(o, list) else [o])
+                return pd.DataFrame(o if isinstance(o, list) else [o]).dropna()
             return self._param.output_var_name, o
 
         outs = None
         for oo in o():
             if not isinstance(oo, pd.DataFrame):
-                outs = pd.DataFrame(oo if isinstance(oo, list) else [oo])
+                outs = pd.DataFrame(oo if isinstance(oo, list) else [oo]).dropna()
             else:
-                outs = oo
+                outs = oo.dropna()
         return self._param.output_var_name, outs
 
     def reset(self):
@@ -455,6 +413,35 @@ class ComponentBase(ABC):
     def set_output(self, v):
         setattr(self._param, self._param.output_var_name, v)
 
+    def set_infor(self, v):
+        setattr(self._param, self._param.infor_var_name, v)
+
+    def _fetch_outputs_from(self, sources: list[dict[str, Any]]) -> list[pd.DataFrame]:
+        outs = []
+        for q in sources:
+            if q.get("component_id"):
+                if "@" in q["component_id"] and q["component_id"].split("@")[0].lower().find("begin") >= 0:
+                    cpn_id, key = q["component_id"].split("@")
+                    for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
+                        if p["key"] == key:
+                            outs.append(pd.DataFrame([{"content": p.get("value", "")}]))
+                            break
+                    else:
+                        assert False, f"Can't find parameter '{key}' for {cpn_id}"
+                    continue
+
+                if q["component_id"].lower().find("answer") == 0:
+                    txt = []
+                    for r, c in self._canvas.history[::-1][: self._param.message_history_window_size][::-1]:
+                        txt.append(f"{r.upper()}:{c}")
+                    txt = "\n".join(txt)
+                    outs.append(pd.DataFrame([{"content": txt}]))
+                    continue
+
+                outs.append(self._canvas.get_component(q["component_id"])["obj"].output(allow_partial=False)[1])
+            elif q.get("value"):
+                outs.append(pd.DataFrame([{"content": q["value"]}]))
+        return outs
     def get_input(self):
         if self._param.debug_inputs:
             return pd.DataFrame([{"content": v["value"]} for v in self._param.debug_inputs if v.get("value")])
@@ -463,49 +450,33 @@ class ComponentBase(ABC):
         if len(self._canvas.path) > 1:
             reversed_cpnts.extend(self._canvas.path[-2])
         reversed_cpnts.extend(self._canvas.path[-1])
+        up_cpns = self.get_upstream()
+        reversed_up_cpnts = [cpn for cpn in reversed_cpnts if cpn in up_cpns]
 
         if self._param.query:
             self._param.inputs = []
-            outs = []
-            for q in self._param.query:
-                if q.get("component_id"):
-                    if q["component_id"].split("@")[0].lower().find("begin") >= 0:
-                        cpn_id, key = q["component_id"].split("@")
-                        for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
-                            if p["key"] == key:
-                                outs.append(pd.DataFrame([{"content": p.get("value", "")}]))
-                                self._param.inputs.append({"component_id": q["component_id"],
-                                                           "content": p.get("value", "")})
-                                break
-                        else:
-                            assert False, f"Can't find parameter '{key}' for {cpn_id}"
-                        continue
+            outs = self._fetch_outputs_from(self._param.query)
 
-                    if q["component_id"].lower().find("answer") == 0:
-                        txt = []
-                        for r, c in self._canvas.history[::-1][:self._param.message_history_window_size][::-1]:
-                            txt.append(f"{r.upper()}: {c}")
-                        txt = "\n".join(txt)
-                        self._param.inputs.append({"content": txt, "component_id": q["component_id"]})
-                        outs.append(pd.DataFrame([{"content": txt}]))
-                        continue
+            for out in outs:
+                records = out.to_dict("records")
+                content: str
+
+                if len(records) > 1:
+                    content = "\n".join([str(d["content"]) for d in records])
+                else:
+                    content = records[0]["content"]
+
+                self._param.inputs.append({"component_id": records[0].get("component_id"), "content": content})
 
-                    outs.append(self._canvas.get_component(q["component_id"])["obj"].output(allow_partial=False)[1])
-                    self._param.inputs.append({"component_id": q["component_id"],
-                                               "content": "\n".join(
-                                                   [str(d["content"]) for d in outs[-1].to_dict('records')])})
-                elif q.get("value"):
-                    self._param.inputs.append({"component_id": None, "content": q["value"]})
-                    outs.append(pd.DataFrame([{"content": q["value"]}]))
             if outs:
                 df = pd.concat(outs, ignore_index=True)
                 if "content" in df:
-                    df = df.drop_duplicates(subset=['content']).reset_index(drop=True)
+                    df = df.drop_duplicates(subset=["content"]).reset_index(drop=True)
                 return df
 
         upstream_outs = []
 
-        for u in reversed_cpnts[::-1]:
+        for u in reversed_up_cpnts[::-1]:
             if self.get_component_name(u) in ["switch", "concentrator"]:
                 continue
             if self.component_name.lower() == "generate" and self.get_component_name(u) == "retrieval":
@@ -514,9 +485,8 @@ class ComponentBase(ABC):
                     o["component_id"] = u
                     upstream_outs.append(o)
                     continue
-            #if self.component_name.lower()!="answer" and u not in self._canvas.get_component(self._id)["upstream"]: continue
-            if self.component_name.lower().find("switch") < 0 \
-                    and self.get_component_name(u) in ["relevant", "categorize"]:
+            # if self.component_name.lower()!="answer" and u not in self._canvas.get_component(self._id)["upstream"]: continue
+            if self.component_name.lower().find("switch") < 0 and self.get_component_name(u) in ["relevant", "categorize"]:
                 continue
             if u.lower().find("answer") >= 0:
                 for r, c in self._canvas.history[::-1]:
@@ -536,7 +506,7 @@ class ComponentBase(ABC):
 
         df = pd.concat(upstream_outs, ignore_index=True)
         if "content" in df:
-            df = df.drop_duplicates(subset=['content']).reset_index(drop=True)
+            df = df.drop_duplicates(subset=["content"]).reset_index(drop=True)
 
         self._param.inputs = []
         for _, r in df.iterrows():
@@ -545,7 +515,7 @@ class ComponentBase(ABC):
         return df
 
     def get_input_elements(self):
-        assert self._param.query, "Please identify input parameters firstly."
+        assert self._param.query, "Please verify the input parameters first."
         eles = []
         for q in self._param.query:
             if q.get("component_id"):
@@ -555,7 +525,7 @@ class ComponentBase(ABC):
                     eles.extend(self._canvas.get_component(cpn_id)["obj"]._param.query)
                     continue
 
-                eles.append({"name": self._canvas.get_compnent_name(cpn_id), "key": cpn_id})
+                eles.append({"name": self._canvas.get_component_name(cpn_id), "key": cpn_id})
             else:
                 eles.append({"key": q["value"], "name": q["value"], "value": q["value"]})
         return eles
@@ -565,8 +535,10 @@ class ComponentBase(ABC):
         if len(self._canvas.path) > 1:
             reversed_cpnts.extend(self._canvas.path[-2])
         reversed_cpnts.extend(self._canvas.path[-1])
+        up_cpns = self.get_upstream()
+        reversed_up_cpnts = [cpn for cpn in reversed_cpnts if cpn in up_cpns]
 
-        for u in reversed_cpnts[::-1]:
+        for u in reversed_up_cpnts[::-1]:
             if self.get_component_name(u) in ["switch", "answer"]:
                 continue
             return self._canvas.get_component(u)["obj"].output()[1]
@@ -584,3 +556,7 @@ class ComponentBase(ABC):
     def get_parent(self):
         pid = self._canvas.get_component(self._id)["parent_id"]
         return self._canvas.get_component(pid)["obj"]
+
+    def get_upstream(self):
+        cpn_nms = self._canvas.get_component(self._id)["upstream"]
+        return cpn_nms

agent/component/categorize.py

@@ -50,26 +50,29 @@ class CategorizeParam(GenerateParam):
         for c, desc in self.category_description.items():
             if desc.get("description"):
                 descriptions.append(
-                    "--------------------\nCategory: {}\nDescription: {}\n".format(c, desc["description"]))
+                    "\nCategory: {}\nDescription: {}".format(c, desc["description"]))
 
         self.prompt = """
-        You're a text classifier. You need to categorize the user’s questions into {} categories, 
-        namely: {}
-        Here's description of each category:
-        {}
+Role: You're a text classifier. 
+Task: You need to categorize the user’s questions into {} categories, namely: {}
 
-        You could learn from the following examples:
-        {}
-        You could learn from the above examples.
-        Just mention the category names, no need for any additional words.
-        
-        ---- Real Data ----
-        {}
+Here's description of each category:
+{}
+
+You could learn from the following examples:
+{}
+You could learn from the above examples.
+
+Requirements:
+- Just mention the category names, no need for any additional words.
+
+---- Real Data ----
+USER: {}\n
         """.format(
             len(self.category_description.keys()),
             "/".join(list(self.category_description.keys())),
             "\n".join(descriptions),
-            "- ".join(cate_lines),
+            "\n\n- ".join(cate_lines),
             chat_hist
         )
         return self.prompt
@@ -80,18 +83,32 @@ class Categorize(Generate, ABC):
 
     def _run(self, history, **kwargs):
         input = self.get_input()
+        input = " - ".join(input["content"]) if "content" in input else ""
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
+        self._canvas.set_component_infor(self._id, {"prompt":self._param.get_prompt(input),"messages":  [{"role": "user", "content": "\nCategory: "}],"conf": self._param.gen_conf()})
+
         ans = chat_mdl.chat(self._param.get_prompt(input), [{"role": "user", "content": "\nCategory: "}],
                             self._param.gen_conf())
-        logging.debug(f"input: {input}, answer: {str(ans)}")
+        logging.debug(f"input: {input}, answer: {str(ans)}")    
+        # Count the number of times each category appears in the answer.
+        category_counts = {}
         for c in self._param.category_description.keys():
-            if ans.lower().find(c.lower()) >= 0:
-                return Categorize.be_output(self._param.category_description[c]["to"])
+            count = ans.lower().count(c.lower())
+            category_counts[c] = count
+            
+        # If a category is found, return the category with the highest count.
+        if any(category_counts.values()):
+            max_category = max(category_counts.items(), key=lambda x: x[1])
+            res = Categorize.be_output(self._param.category_description[max_category[0]]["to"])
+            self.set_output(res)
+            return res
 
-        return Categorize.be_output(list(self._param.category_description.items())[-1][1]["to"])
+        res = Categorize.be_output(list(self._param.category_description.items())[-1][1]["to"])
+        self.set_output(res)
+        return res
 
     def debug(self, **kwargs):
         df = self._run([], **kwargs)
         cpn_id = df.iloc[0, 0]
-        return Categorize.be_output(self._canvas.get_compnent_name(cpn_id))
+        return Categorize.be_output(self._canvas.get_component_name(cpn_id))
 

agent/component/code.py

@@ -0,0 +1,152 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import base64
+from abc import ABC
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+from agent.component.base import ComponentBase, ComponentParamBase
+from api import settings
+
+
+class Language(str, Enum):
+    PYTHON = "python"
+    NODEJS = "nodejs"
+
+
+class CodeExecutionRequest(BaseModel):
+    code_b64: str = Field(..., description="Base64 encoded code string")
+    language: Language = Field(default=Language.PYTHON, description="Programming language")
+    arguments: Optional[dict] = Field(default={}, description="Arguments")
+
+    @field_validator("code_b64")
+    @classmethod
+    def validate_base64(cls, v: str) -> str:
+        try:
+            base64.b64decode(v, validate=True)
+            return v
+        except Exception as e:
+            raise ValueError(f"Invalid base64 encoding: {str(e)}")
+
+    @field_validator("language", mode="before")
+    @classmethod
+    def normalize_language(cls, v) -> str:
+        if isinstance(v, str):
+            low = v.lower()
+            if low in ("python", "python3"):
+                return "python"
+            elif low in ("javascript", "nodejs"):
+                return "nodejs"
+        raise ValueError(f"Unsupported language: {v}")
+
+
+class CodeParam(ComponentParamBase):
+    """
+    Define the code sandbox component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.lang = "python"
+        self.script = ""
+        self.arguments = []
+        self.address = f"http://{settings.SANDBOX_HOST}:9385/run"
+        self.enable_network = True
+
+    def check(self):
+        self.check_valid_value(self.lang, "Support languages", ["python", "python3", "nodejs", "javascript"])
+        self.check_defined_type(self.enable_network, "Enable network", ["bool"])
+
+
+class Code(ComponentBase, ABC):
+    component_name = "Code"
+
+    def _run(self, history, **kwargs):
+        arguments = {}
+        for input in self._param.arguments:
+            if "@" in input["component_id"]:
+                component_id = input["component_id"].split("@")[0]
+                referred_component_key = input["component_id"].split("@")[1]
+                referred_component = self._canvas.get_component(component_id)["obj"]
+
+                for param in referred_component._param.query:
+                    if param["key"] == referred_component_key:
+                        if "value" in param:
+                            arguments[input["name"]] = param["value"]
+            else:
+                referred_component = self._canvas.get_component(input["component_id"])["obj"]
+                referred_component_name = referred_component.component_name
+                referred_component_id = referred_component._id
+
+                debug_inputs = self._param.debug_inputs
+                if debug_inputs:
+                    for param in debug_inputs:
+                        if param["key"] == referred_component_id:
+                            if "value" in param and param["name"] == input["name"]:
+                                arguments[input["name"]] = param["value"]
+                else:
+                    if referred_component_name.lower() == "answer":
+                        arguments[input["name"]] = self._canvas.get_history(1)[0]["content"]
+                        continue
+
+                    _, out = referred_component.output(allow_partial=False)
+                    if not out.empty:
+                        arguments[input["name"]] = "\n".join(out["content"])
+
+        return self._execute_code(
+            language=self._param.lang,
+            code=self._param.script,
+            arguments=arguments,
+            address=self._param.address,
+            enable_network=self._param.enable_network,
+        )
+
+    def _execute_code(self, language: str, code: str, arguments: dict, address: str, enable_network: bool):
+        import requests
+
+        try:
+            code_b64 = self._encode_code(code)
+            code_req = CodeExecutionRequest(code_b64=code_b64, language=language, arguments=arguments).model_dump()
+        except Exception as e:
+            return Code.be_output("**Error**: construct code request error: " + str(e))
+
+        try:
+            resp = requests.post(url=address, json=code_req, timeout=10)
+            body = resp.json()
+            if body:
+                stdout = body.get("stdout")
+                stderr = body.get("stderr")
+                return Code.be_output(stdout or stderr)
+            else:
+                return Code.be_output("**Error**: There is no response from sanbox")
+
+        except Exception as e:
+            return Code.be_output("**Error**: Internal error in sanbox: " + str(e))
+
+    def _encode_code(self, code: str) -> str:
+        return base64.b64encode(code.encode("utf-8")).decode("utf-8")
+
+    def get_input_elements(self):
+        elements = []
+        for input in self._param.arguments:
+            cpn_id = input["component_id"]
+            elements.append({"key": cpn_id, "name": input["name"]})
+        return elements
+
+    def debug(self, **kwargs):
+        return self._run([], **kwargs)

agent/component/email.py

@@ -82,7 +82,10 @@ class Email(ComponentBase, ABC):
             logging.info(f"Connecting to SMTP server {self._param.smtp_server}:{self._param.smtp_port}")
             
             context = smtplib.ssl.create_default_context()
-            with smtplib.SMTP_SSL(self._param.smtp_server, self._param.smtp_port, context=context) as server:
+            with smtplib.SMTP(self._param.smtp_server, self._param.smtp_port) as server:
+                server.ehlo()
+                server.starttls(context=context)
+                server.ehlo()
                 # Login
                 logging.info(f"Attempting to login with email: {self._param.email}")
                 server.login(self._param.email, self._param.password)

agent/component/exesql.py

@@ -52,15 +52,16 @@ class ExeSQLParam(GenerateParam):
         self.check_positive_integer(self.top_n, "Number of records")
         if self.database == "rag_flow":
             if self.host == "ragflow-mysql":
-                raise ValueError("The host is not accessible.")
+                raise ValueError("For the security reason, it dose not support database named rag_flow.")
             if self.password == "infini_rag_flow":
-                raise ValueError("The host is not accessible.")
+                raise ValueError("For the security reason, it dose not support database named rag_flow.")
 
 
 class ExeSQL(Generate, ABC):
     component_name = "ExeSQL"
 
-    def _refactor(self,ans):
+    def _refactor(self, ans):
+        ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
         match = re.search(r"```sql\s*(.*?)\s*```", ans, re.DOTALL)
         if match:
             ans = match.group(1)  # Query content
@@ -78,7 +79,6 @@ class ExeSQL(Generate, ABC):
         ans = self.get_input()
         ans = "".join([str(a) for a in ans["content"]]) if "content" in ans else ""
         ans = self._refactor(ans)
-        logging.info("db_type: ",self._param.db_type)
         if self._param.db_type in ["mysql", "mariadb"]:
             db = pymysql.connect(db=self._param.database, user=self._param.username, host=self._param.host,
                                  port=self._param.port, password=self._param.password)
@@ -87,11 +87,11 @@ class ExeSQL(Generate, ABC):
                                   port=self._param.port, password=self._param.password)
         elif self._param.db_type == 'mssql':
             conn_str = (
-                r'DRIVER={ODBC Driver 17 for SQL Server};'
-                r'SERVER=' + self._param.host + ',' + str(self._param.port) + ';'
-                r'DATABASE=' + self._param.database + ';'
-                r'UID=' + self._param.username + ';'
-                r'PWD=' + self._param.password
+                    r'DRIVER={ODBC Driver 17 for SQL Server};'
+                    r'SERVER=' + self._param.host + ',' + str(self._param.port) + ';'
+                    r'DATABASE=' + self._param.database + ';'
+                    r'UID=' + self._param.username + ';'
+                    r'PWD=' + self._param.password
             )
             db = pyodbc.connect(conn_str)
         try:
@@ -101,51 +101,51 @@ class ExeSQL(Generate, ABC):
         if not hasattr(self, "_loop"):
             setattr(self, "_loop", 0)
             self._loop += 1
-        input_list=re.split(r';', ans.replace(r"\n", " "))
+        input_list = re.split(r';', ans.replace(r"\n", " "))
         sql_res = []
         for i in range(len(input_list)):
-            single_sql=input_list[i]
+            single_sql = input_list[i]
+            single_sql = single_sql.replace('```','')
             while self._loop <= self._param.loop:
-                self._loop+=1
+                self._loop += 1
                 if not single_sql:
                     break
                 try:
-                    logging.info("single_sql: ", single_sql)
                     cursor.execute(single_sql)
                     if cursor.rowcount == 0:
                         sql_res.append({"content": "No record in the database!"})
                         break
                     if self._param.db_type == 'mssql':
-                        single_res  = pd.DataFrame.from_records(cursor.fetchmany(self._param.top_n),columns = [desc[0] for desc in cursor.description])
+                        single_res = pd.DataFrame.from_records(cursor.fetchmany(self._param.top_n),
+                                                               columns=[desc[0] for desc in cursor.description])
                     else:
                         single_res = pd.DataFrame([i for i in cursor.fetchmany(self._param.top_n)])
                         single_res.columns = [i[0] for i in cursor.description]
-                    sql_res.append({"content":  single_res.to_markdown()})
+                    sql_res.append({"content": single_res.to_markdown(index=False, floatfmt=".6f")})
                     break
                 except Exception as e:
                     single_sql = self._regenerate_sql(single_sql, str(e), **kwargs)
                     single_sql = self._refactor(single_sql)
                     if self._loop > self._param.loop:
                         sql_res.append({"content": "Can't query the correct data via SQL statement."})
-                        # raise Exception("Maximum loop time exceeds. Can't query the correct data via SQL statement.")
         db.close()
         if not sql_res:
             return ExeSQL.be_output("")
         return pd.DataFrame(sql_res)
 
-    def _regenerate_sql(self, failed_sql, error_message,**kwargs):
+    def _regenerate_sql(self, failed_sql, error_message, **kwargs):
         prompt = f'''
         ## You are the Repair SQL Statement Helper, please modify the original SQL statement based on the SQL query error report.
         ## The original SQL statement is as follows:{failed_sql}.
         ## The contents of the SQL query error report is as follows:{error_message}.
         ## Answer only the modified SQL statement. Please do not give any explanation, just answer the code.
 '''
-        self._param.prompt=prompt
+        self._param.prompt = prompt
         kwargs_ = deepcopy(kwargs)
         kwargs_["stream"] = False
         response = Generate._run(self, [], **kwargs_)
         try:
-            regenerated_sql = response.loc[0,"content"]
+            regenerated_sql = response.loc[0, "content"]
             return regenerated_sql
         except Exception as e:
             logging.error(f"Failed to regenerate SQL: {e}")

agent/component/generate.py

@@ -13,15 +13,30 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import json
 import re
 from functools import partial
+from typing import Any
 import pandas as pd
 from api.db import LLMType
 from api.db.services.conversation_service import structure_answer
-from api.db.services.dialog_service import message_fit_in
 from api.db.services.llm_service import LLMBundle
 from api import settings
 from agent.component.base import ComponentBase, ComponentParamBase
+from plugin import GlobalPluginManager
+from plugin.llm_tool_plugin import llm_tool_metadata_to_openai_tool
+from rag.llm.chat_model import ToolCallSession
+from rag.prompts import message_fit_in
+
+
+class LLMToolPluginCallSession(ToolCallSession):
+    def tool_call(self, name: str, arguments: dict[str, Any]) -> str:
+        tool = GlobalPluginManager.get_llm_tool_by_name(name)
+
+        if tool is None:
+            raise ValueError(f"LLM tool {name} does not exist")
+
+        return tool().invoke(**arguments)
 
 
 class GenerateParam(ComponentParamBase):
@@ -40,6 +55,7 @@ class GenerateParam(ComponentParamBase):
         self.frequency_penalty = 0
         self.cite = True
         self.parameters = []
+        self.llm_enabled_tools = []
 
     def check(self):
         self.check_decimal_float(self.temperature, "[Generate] Temperature")
@@ -69,36 +85,35 @@ class Generate(ComponentBase):
     component_name = "Generate"
 
     def get_dependent_components(self):
-        cpnts = set([para["component_id"].split("@")[0] for para in self._param.parameters \
-                     if para.get("component_id") \
-                     and para["component_id"].lower().find("answer") < 0 \
-                     and para["component_id"].lower().find("begin") < 0])
+        inputs = self.get_input_elements()
+        cpnts = set([i["key"] for i in inputs[1:] if i["key"].lower().find("answer") < 0 and i["key"].lower().find("begin") < 0])
         return list(cpnts)
 
     def set_cite(self, retrieval_res, answer):
-        retrieval_res = retrieval_res.dropna(subset=["vector", "content_ltks"]).reset_index(drop=True)
         if "empty_response" in retrieval_res.columns:
             retrieval_res["empty_response"].fillna("", inplace=True)
+        chunks = json.loads(retrieval_res["chunks"][0])
         answer, idx = settings.retrievaler.insert_citations(answer,
-                                                            [ck["content_ltks"] for _, ck in retrieval_res.iterrows()],
-                                                            [ck["vector"] for _, ck in retrieval_res.iterrows()],
+                                                            [ck["content_ltks"] for ck in chunks],
+                                                            [ck["vector"] for ck in chunks],
                                                             LLMBundle(self._canvas.get_tenant_id(), LLMType.EMBEDDING,
                                                                       self._canvas.get_embedding_model()), tkweight=0.7,
                                                             vtweight=0.3)
         doc_ids = set([])
         recall_docs = []
         for i in idx:
-            did = retrieval_res.loc[int(i), "doc_id"]
+            did = chunks[int(i)]["doc_id"]
             if did in doc_ids:
                 continue
             doc_ids.add(did)
-            recall_docs.append({"doc_id": did, "doc_name": retrieval_res.loc[int(i), "docnm_kwd"]})
+            recall_docs.append({"doc_id": did, "doc_name": chunks[int(i)]["docnm_kwd"]})
 
-        del retrieval_res["vector"]
-        del retrieval_res["content_ltks"]
+        for c in chunks:
+            del c["vector"]
+            del c["content_ltks"]
 
         reference = {
-            "chunks": [ck.to_dict() for _, ck in retrieval_res.iterrows()],
+            "chunks": chunks,
             "doc_aggs": recall_docs
         }
 
@@ -110,33 +125,56 @@ class Generate(ComponentBase):
         return res
 
     def get_input_elements(self):
-        if self._param.parameters:
-            return [{"key": "user", "name": "Input your question here:"}, *self._param.parameters]
-
-        return [{"key": "user", "name": "Input your question here:"}]
+        key_set = set([])
+        res = [{"key": "user", "name": "Input your question here:"}]
+        for r in re.finditer(r"\{([a-z]+[:@][a-z0-9_-]+)\}", self._param.prompt, flags=re.IGNORECASE):
+            cpn_id = r.group(1)
+            if cpn_id in key_set:
+                continue
+            if cpn_id.lower().find("begin@") == 0:
+                cpn_id, key = cpn_id.split("@")
+                for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
+                    if p["key"] != key:
+                        continue
+                    res.append({"key": r.group(1), "name": p["name"]})
+                    key_set.add(r.group(1))
+                continue
+            cpn_nm = self._canvas.get_component_name(cpn_id)
+            if not cpn_nm:
+                continue
+            res.append({"key": cpn_id, "name": cpn_nm})
+            key_set.add(cpn_id)
+        return res
 
     def _run(self, history, **kwargs):
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
+
+        if len(self._param.llm_enabled_tools) > 0:
+            tools = GlobalPluginManager.get_llm_tools_by_names(self._param.llm_enabled_tools)
+
+            chat_mdl.bind_tools(
+                LLMToolPluginCallSession(),
+                [llm_tool_metadata_to_openai_tool(t.get_metadata()) for t in tools]
+            )
+
         prompt = self._param.prompt
 
         retrieval_res = []
         self._param.inputs = []
-        for para in self._param.parameters:
-            if not para.get("component_id"):
-                continue
-            component_id = para["component_id"].split("@")[0]
-            if para["component_id"].lower().find("@") >= 0:
-                cpn_id, key = para["component_id"].split("@")
+        for para in self.get_input_elements()[1:]:
+            if para["key"].lower().find("begin@") == 0:
+                cpn_id, key = para["key"].split("@")
                 for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
                     if p["key"] == key:
                         kwargs[para["key"]] = p.get("value", "")
                         self._param.inputs.append(
-                            {"component_id": para["component_id"], "content": kwargs[para["key"]]})
+                            {"component_id": para["key"], "content": kwargs[para["key"]]})
                         break
                 else:
                     assert False, f"Can't find parameter '{key}' for {cpn_id}"
                 continue
 
+            component_id = para["key"]
             cpn = self._canvas.get_component(component_id)["obj"]
             if cpn.component_name.lower() == "answer":
                 hist = self._canvas.get_history(1)
@@ -152,8 +190,8 @@ class Generate(ComponentBase):
             else:
                 if cpn.component_name.lower() == "retrieval":
                     retrieval_res.append(out)
-                kwargs[para["key"]] = "  - "+"\n - ".join([o if isinstance(o, str) else str(o) for o in out["content"]])
-            self._param.inputs.append({"component_id": para["component_id"], "content": kwargs[para["key"]]})
+                kwargs[para["key"]] = "  - " + "\n - ".join([o if isinstance(o, str) else str(o) for o in out["content"]])
+            self._param.inputs.append({"component_id": para["key"], "content": kwargs[para["key"]]})
 
         if retrieval_res:
             retrieval_res = pd.concat(retrieval_res, ignore_index=True)
@@ -175,19 +213,20 @@ class Generate(ComponentBase):
             return partial(self.stream_output, chat_mdl, prompt, retrieval_res)
 
         if "empty_response" in retrieval_res.columns and not "".join(retrieval_res["content"]):
-            res = {"content": "\n- ".join(retrieval_res["empty_response"]) if "\n- ".join(
-                retrieval_res["empty_response"]) else "Nothing found in knowledgebase!", "reference": []}
+            empty_res = "\n- ".join([str(t) for t in retrieval_res["empty_response"] if str(t)])
+            res = {"content": empty_res if empty_res else "Nothing found in knowledgebase!", "reference": []}
             return pd.DataFrame([res])
 
         msg = self._canvas.get_history(self._param.message_history_window_size)
         if len(msg) < 1:
-            msg.append({"role": "user", "content": ""})
+            msg.append({"role": "user", "content": "Output: "})
         _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(chat_mdl.max_length * 0.97))
         if len(msg) < 2:
-            msg.append({"role": "user", "content": ""})
+            msg.append({"role": "user", "content": "Output: "})
         ans = chat_mdl.chat(msg[0]["content"], msg[1:], self._param.gen_conf())
-
-        if self._param.cite and "content_ltks" in retrieval_res.columns and "vector" in retrieval_res.columns:
+        ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+        self._canvas.set_component_infor(self._id, {"prompt":msg[0]["content"],"messages":  msg[1:],"conf":  self._param.gen_conf()})
+        if self._param.cite and "chunks" in retrieval_res.columns:
             res = self.set_cite(retrieval_res, ans)
             return pd.DataFrame([res])
 
@@ -196,28 +235,30 @@ class Generate(ComponentBase):
     def stream_output(self, chat_mdl, prompt, retrieval_res):
         res = None
         if "empty_response" in retrieval_res.columns and not "".join(retrieval_res["content"]):
-            res = {"content": "\n- ".join(retrieval_res["empty_response"]) if "\n- ".join(
-                retrieval_res["empty_response"]) else "Nothing found in knowledgebase!", "reference": []}
+            empty_res = "\n- ".join([str(t) for t in retrieval_res["empty_response"] if str(t)])
+            res = {"content": empty_res if empty_res else "Nothing found in knowledgebase!", "reference": []}
             yield res
             self.set_output(res)
             return
 
         msg = self._canvas.get_history(self._param.message_history_window_size)
+        if msg and msg[0]['role'] == 'assistant':
+            msg.pop(0)
         if len(msg) < 1:
-            msg.append({"role": "user", "content": ""})
+            msg.append({"role": "user", "content": "Output: "})
         _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(chat_mdl.max_length * 0.97))
         if len(msg) < 2:
-            msg.append({"role": "user", "content": ""})
+            msg.append({"role": "user", "content": "Output: "})
         answer = ""
         for ans in chat_mdl.chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf()):
             res = {"content": ans, "reference": []}
             answer = ans
             yield res
 
-        if self._param.cite and "content_ltks" in retrieval_res.columns and "vector" in retrieval_res.columns:
+        if self._param.cite and "chunks" in retrieval_res.columns:
             res = self.set_cite(retrieval_res, answer)
             yield res
-
+        self._canvas.set_component_infor(self._id, {"prompt":msg[0]["content"],"messages":  msg[1:],"conf":  self._param.gen_conf()})
         self.set_output(Generate.be_output(res))
 
     def debug(self, **kwargs):
@@ -230,5 +271,6 @@ class Generate(ComponentBase):
         for n, v in kwargs.items():
             prompt = re.sub(r"\{%s\}" % re.escape(n), str(v).replace("\\", " "), prompt)
 
-        ans = chat_mdl.chat(prompt, [{"role": "user", "content": kwargs.get("user", "")}], self._param.gen_conf())
+        u = kwargs.get("user")
+        ans = chat_mdl.chat(prompt, [{"role": "user", "content": u if u else "Output: "}], self._param.gen_conf())
         return pd.DataFrame([ans])

agent/component/invoke.py

@@ -35,12 +35,14 @@ class InvokeParam(ComponentParamBase):
         self.url = ""
         self.timeout = 60
         self.clean_html = False
+        self.datatype = "json"  # New parameter to determine data posting type
 
     def check(self):
         self.check_valid_value(self.method.lower(), "Type of content from the crawler", ['get', 'post', 'put'])
         self.check_empty(self.url, "End point URL")
         self.check_positive_integer(self.timeout, "Timeout time in second")
         self.check_boolean(self.clean_html, "Clean HTML")
+        self.check_valid_value(self.datatype.lower(), "Data post type", ['json', 'formdata'])  # Check for valid datapost value
 
 
 class Invoke(ComponentBase, ABC):
@@ -94,22 +96,36 @@ class Invoke(ComponentBase, ABC):
             return Invoke.be_output(response.text)
 
         if method == 'put':
-            response = requests.put(url=url,
-                                    data=args,
-                                    headers=headers,
-                                    proxies=proxies,
-                                    timeout=self._param.timeout)
+            if self._param.datatype.lower() == 'json':
+                response = requests.put(url=url,
+                                        json=args,
+                                        headers=headers,
+                                        proxies=proxies,
+                                        timeout=self._param.timeout)
+            else:
+                response = requests.put(url=url,
+                                        data=args,
+                                        headers=headers,
+                                        proxies=proxies,
+                                        timeout=self._param.timeout)
             if self._param.clean_html:
                 sections = HtmlParser()(None, response.content)
                 return Invoke.be_output("\n".join(sections))
             return Invoke.be_output(response.text)
 
         if method == 'post':
-            response = requests.post(url=url,
-                                    json=args,
-                                    headers=headers,
-                                    proxies=proxies,
-                                    timeout=self._param.timeout)
+            if self._param.datatype.lower() == 'json':
+                response = requests.post(url=url,
+                                         json=args,
+                                         headers=headers,
+                                         proxies=proxies,
+                                         timeout=self._param.timeout)
+            else:
+                response = requests.post(url=url,
+                                         data=args,
+                                         headers=headers,
+                                         proxies=proxies,
+                                         timeout=self._param.timeout)
             if self._param.clean_html:
                 sections = HtmlParser()(None, response.content)
                 return Invoke.be_output("\n".join(sections))

agent/component/iterationitem.py

@@ -38,6 +38,10 @@ class IterationItem(ComponentBase, ABC):
         ans = parent.get_input()
         ans = parent._param.delimiter.join(ans["content"]) if "content" in ans else ""
         ans = [a.strip() for a in ans.split(parent._param.delimiter)]
+        if not ans:
+            self._idx = -1
+            return pd.DataFrame()
+
         df = pd.DataFrame([{"content": ans[self._idx]}])
         self._idx += 1
         if self._idx >= len(ans):

agent/component/keyword.py

@@ -51,15 +51,22 @@ class KeywordExtract(Generate, ABC):
 
     def _run(self, history, **kwargs):
         query = self.get_input()
-        query = str(query["content"][0]) if "content" in query else ""
+        if hasattr(query, "to_dict") and "content" in query:
+            query = ", ".join(map(str, query["content"].dropna()))
+        else:
+            query = str(query)
+
 
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
+        self._canvas.set_component_infor(self._id, {"prompt":self._param.get_prompt(),"messages":  [{"role": "user", "content": query}],"conf": self._param.gen_conf()})
+
         ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": query}],
                             self._param.gen_conf())
 
+        ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
         ans = re.sub(r".*keyword:", "", ans).strip()
         logging.debug(f"ans: {ans}")
         return KeywordExtract.be_output(ans)
 
     def debug(self, **kwargs):
-        return self._run([], **kwargs)
+        return self._run([], **kwargs)

agent/component/message.py

@@ -40,7 +40,9 @@ class Message(ComponentBase, ABC):
         if kwargs.get("stream"):
             return partial(self.stream_output)
 
-        return Message.be_output(random.choice(self._param.messages))
+        res = Message.be_output(random.choice(self._param.messages))
+        self.set_output(res)
+        return res
 
     def stream_output(self):
         res = None

agent/component/retrieval.py

@@ -13,24 +13,28 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import json
 import logging
+import re
 from abc import ABC
 
 import pandas as pd
 
 from api.db import LLMType
-from api.db.services.dialog_service import label_question
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api import settings
 from agent.component.base import ComponentBase, ComponentParamBase
+from rag.app.tag import label_question
+from rag.prompts import kb_prompt
+from rag.utils.tavily_conn import Tavily
 
 
 class RetrievalParam(ComponentParamBase):
-
     """
     Define the Retrieval component parameters.
     """
+
     def __init__(self):
         super().__init__()
         self.similarity_threshold = 0.2
@@ -38,12 +42,15 @@ class RetrievalParam(ComponentParamBase):
         self.top_n = 8
         self.top_k = 1024
         self.kb_ids = []
+        self.kb_vars = []
         self.rerank_id = ""
         self.empty_response = ""
+        self.tavily_api_key = ""
+        self.use_kg = False
 
     def check(self):
         self.check_decimal_float(self.similarity_threshold, "[Retrieval] Similarity threshold")
-        self.check_decimal_float(self.keywords_similarity_weight, "[Retrieval] Keywords similarity weight")
+        self.check_decimal_float(self.keywords_similarity_weight, "[Retrieval] Keyword similarity weight")
         self.check_positive_number(self.top_n, "[Retrieval] Top N")
 
 
@@ -53,26 +60,69 @@ class Retrieval(ComponentBase, ABC):
     def _run(self, history, **kwargs):
         query = self.get_input()
         query = str(query["content"][0]) if "content" in query else ""
+        query = re.split(r"(USER:|ASSISTANT:)", query)[-1]
 
-        kbs = KnowledgebaseService.get_by_ids(self._param.kb_ids)
+        kb_ids: list[str] = self._param.kb_ids or []
+
+        kb_vars = self._fetch_outputs_from(self._param.kb_vars)
+
+        if len(kb_vars) > 0:
+            for kb_var in kb_vars:
+                if len(kb_var) == 1:
+                    kb_var_value = str(kb_var["content"][0])
+
+                    for v in kb_var_value.split(","):
+                        kb_ids.append(v)
+                else:
+                    for v in kb_var.to_dict("records"):
+                        kb_ids.append(v["content"])
+
+        filtered_kb_ids: list[str] = [kb_id for kb_id in kb_ids if kb_id]
+
+        kbs = KnowledgebaseService.get_by_ids(filtered_kb_ids)
         if not kbs:
             return Retrieval.be_output("")
 
         embd_nms = list(set([kb.embd_id for kb in kbs]))
         assert len(embd_nms) == 1, "Knowledge bases use different embedding models."
 
-        embd_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.EMBEDDING, embd_nms[0])
-        self._canvas.set_embedding_model(embd_nms[0])
+        embd_mdl = None
+        if embd_nms:
+            embd_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.EMBEDDING, embd_nms[0])
+            self._canvas.set_embedding_model(embd_nms[0])
 
         rerank_mdl = None
         if self._param.rerank_id:
             rerank_mdl = LLMBundle(kbs[0].tenant_id, LLMType.RERANK, self._param.rerank_id)
 
-        kbinfos = settings.retrievaler.retrieval(query, embd_mdl, kbs[0].tenant_id, self._param.kb_ids,
-                                        1, self._param.top_n,
-                                        self._param.similarity_threshold, 1 - self._param.keywords_similarity_weight,
-                                        aggs=False, rerank_mdl=rerank_mdl,
-                                        rank_feature=label_question(query, kbs))
+        if kbs:
+            query = re.sub(r"^user[:：\s]*", "", query, flags=re.IGNORECASE)
+            kbinfos = settings.retrievaler.retrieval(
+                query,
+                embd_mdl,
+                [kb.tenant_id for kb in kbs],
+                filtered_kb_ids,
+                1,
+                self._param.top_n,
+                self._param.similarity_threshold,
+                1 - self._param.keywords_similarity_weight,
+                aggs=False,
+                rerank_mdl=rerank_mdl,
+                rank_feature=label_question(query, kbs),
+            )
+        else:
+            kbinfos = {"chunks": [], "doc_aggs": []}
+
+        if self._param.use_kg and kbs:
+            ck = settings.kg_retrievaler.retrieval(query, [kb.tenant_id for kb in kbs], filtered_kb_ids, embd_mdl, LLMBundle(kbs[0].tenant_id, LLMType.CHAT))
+            if ck["content_with_weight"]:
+                kbinfos["chunks"].insert(0, ck)
+
+        if self._param.tavily_api_key:
+            tav = Tavily(self._param.tavily_api_key)
+            tav_res = tav.retrieve_chunks(query)
+            kbinfos["chunks"].extend(tav_res["chunks"])
+            kbinfos["doc_aggs"].extend(tav_res["doc_aggs"])
 
         if not kbinfos["chunks"]:
             df = Retrieval.be_output("")
@@ -80,10 +130,6 @@ class Retrieval(ComponentBase, ABC):
                 df["empty_response"] = self._param.empty_response
             return df
 
-        df = pd.DataFrame(kbinfos["chunks"])
-        df["content"] = df["content_with_weight"]
-        del df["content_with_weight"]
+        df = pd.DataFrame({"content": kb_prompt(kbinfos, 200000), "chunks": json.dumps(kbinfos["chunks"])})
         logging.debug("{} {}".format(query, df))
-        return df
-
-
+        return df.dropna()

agent/component/rewrite.py

@@ -13,93 +13,82 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import logging
 from abc import ABC
-from api.db import LLMType
-from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
+from rag.prompts import full_question
 
 
 class RewriteQuestionParam(GenerateParam):
-
     """
     Define the QuestionRewrite component parameters.
     """
+
     def __init__(self):
         super().__init__()
         self.temperature = 0.9
         self.prompt = ""
+        self.language = ""
 
     def check(self):
         super().check()
 
-    def get_prompt(self, conv):
-        self.prompt = """
-        You are an expert at query expansion to generate a paraphrasing of a question.
-        I can't retrieval relevant information from the knowledge base by using user's question directly.     
-        You need to expand or paraphrase user's question by multiple ways such as using synonyms words/phrase, 
-        writing the abbreviation in its entirety, adding some extra descriptions or explanations, 
-        changing the way of expression, translating the original question into another language (English/Chinese), etc. 
-        And return 5 versions of question and one is from translation.
-        Just list the question. No other words are needed.
-        """
-        return f"""
-Role: A helpful assistant
-Task: Generate a full user question that would follow the conversation.
-Requirements & Restrictions:
-  - Text generated MUST be in the same language of the original user's question.
-  - If the user's latest question is completely, don't do anything, just return the original question.
-  - DON'T generate anything except a refined question.
-
-######################
--Examples-
-######################
-# Example 1
-## Conversation
-USER: What is the name of Donald Trump's father?
-ASSISTANT:  Fred Trump.
-USER: And his mother?
-###############
-Output: What's the name of Donald Trump's mother?
-------------
-# Example 2
-## Conversation
-USER: What is the name of Donald Trump's father?
-ASSISTANT:  Fred Trump.
-USER: And his mother?
-ASSISTANT:  Mary Trump.
-User: What's her full name?
-###############
-Output: What's the full name of Donald Trump's mother Mary Trump?
-######################
-# Real Data
-## Conversation
-{conv}
-###############
-    """
-        return self.prompt
-
 
 class RewriteQuestion(Generate, ABC):
     component_name = "RewriteQuestion"
 
     def _run(self, history, **kwargs):
         hist = self._canvas.get_history(self._param.message_history_window_size)
-        conv = []
-        for m in hist:
-            if m["role"] not in ["user", "assistant"]:
-                continue
-            conv.append("{}: {}".format(m["role"].upper(), m["content"]))
-        conv = "\n".join(conv)
-
-        chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
-        ans = chat_mdl.chat(self._param.get_prompt(conv), [{"role": "user", "content": "Output: "}],
-                            self._param.gen_conf())
+        query = self.get_input()
+        query = str(query["content"][0]) if "content" in query else ""
+        messages = [h for h in hist if h["role"]!="system"]
+        if messages[-1]["role"] != "user":
+            messages.append({"role": "user", "content": query})
+        ans = full_question(self._canvas.get_tenant_id(), self._param.llm_id, messages, self.gen_lang(self._param.language))
         self._canvas.history.pop()
         self._canvas.history.append(("user", ans))
-
-        logging.debug(ans)
         return RewriteQuestion.be_output(ans)
 
-
-
+    @staticmethod
+    def gen_lang(language):
+        # convert code lang to language word for the prompt
+        language_dict = {'af': 'Afrikaans', 'ak': 'Akan', 'sq': 'Albanian', 'ws': 'Samoan', 'am': 'Amharic',
+                         'ar': 'Arabic', 'hy': 'Armenian', 'az': 'Azerbaijani', 'eu': 'Basque', 'be': 'Belarusian',
+                         'bem': 'Bemba', 'bn': 'Bengali', 'bh': 'Bihari',
+                         'xx-bork': 'Bork', 'bs': 'Bosnian', 'br': 'Breton', 'bg': 'Bulgarian', 'bt': 'Bhutani',
+                         'km': 'Cambodian', 'ca': 'Catalan', 'chr': 'Cherokee', 'ny': 'Chichewa', 'zh-cn': 'Chinese',
+                         'zh-tw': 'Chinese', 'co': 'Corsican',
+                         'hr': 'Croatian', 'cs': 'Czech', 'da': 'Danish', 'nl': 'Dutch', 'xx-elmer': 'Elmer',
+                         'en': 'English', 'eo': 'Esperanto', 'et': 'Estonian', 'ee': 'Ewe', 'fo': 'Faroese',
+                         'tl': 'Filipino', 'fi': 'Finnish', 'fr': 'French',
+                         'fy': 'Frisian', 'gaa': 'Ga', 'gl': 'Galician', 'ka': 'Georgian', 'de': 'German',
+                         'el': 'Greek', 'kl': 'Greenlandic', 'gn': 'Guarani', 'gu': 'Gujarati', 'xx-hacker': 'Hacker',
+                         'ht': 'Haitian Creole', 'ha': 'Hausa', 'haw': 'Hawaiian',
+                         'iw': 'Hebrew', 'hi': 'Hindi', 'hu': 'Hungarian', 'is': 'Icelandic', 'ig': 'Igbo',
+                         'id': 'Indonesian', 'ia': 'Interlingua', 'ga': 'Irish', 'it': 'Italian', 'ja': 'Japanese',
+                         'jw': 'Javanese', 'kn': 'Kannada', 'kk': 'Kazakh', 'rw': 'Kinyarwanda',
+                         'rn': 'Kirundi', 'xx-klingon': 'Klingon', 'kg': 'Kongo', 'ko': 'Korean', 'kri': 'Krio',
+                         'ku': 'Kurdish', 'ckb': 'Kurdish (Sorani)', 'ky': 'Kyrgyz', 'lo': 'Laothian', 'la': 'Latin',
+                         'lv': 'Latvian', 'ln': 'Lingala', 'lt': 'Lithuanian',
+                         'loz': 'Lozi', 'lg': 'Luganda', 'ach': 'Luo', 'mk': 'Macedonian', 'mg': 'Malagasy',
+                         'ms': 'Malay', 'ml': 'Malayalam', 'mt': 'Maltese', 'mv': 'Maldivian', 'mi': 'Maori',
+                         'mr': 'Marathi', 'mfe': 'Mauritian Creole', 'mo': 'Moldavian', 'mn': 'Mongolian',
+                         'sr-me': 'Montenegrin', 'my': 'Burmese', 'ne': 'Nepali', 'pcm': 'Nigerian Pidgin',
+                         'nso': 'Northern Sotho', 'no': 'Norwegian', 'nn': 'Norwegian Nynorsk', 'oc': 'Occitan',
+                         'or': 'Oriya', 'om': 'Oromo', 'ps': 'Pashto', 'fa': 'Persian',
+                         'xx-pirate': 'Pirate', 'pl': 'Polish', 'pt': 'Portuguese', 'pt-br': 'Portuguese (Brazilian)',
+                         'pt-pt': 'Portuguese (Portugal)', 'pa': 'Punjabi', 'qu': 'Quechua', 'ro': 'Romanian',
+                         'rm': 'Romansh', 'nyn': 'Runyankole', 'ru': 'Russian', 'gd': 'Scots Gaelic',
+                         'sr': 'Serbian', 'sh': 'Serbo-Croatian', 'st': 'Sesotho', 'tn': 'Setswana',
+                         'crs': 'Seychellois Creole', 'sn': 'Shona', 'sd': 'Sindhi', 'si': 'Sinhalese', 'sk': 'Slovak',
+                         'sl': 'Slovenian', 'so': 'Somali', 'es': 'Spanish', 'es-419': 'Spanish (Latin America)',
+                         'su': 'Sundanese',
+                         'sw': 'Swahili', 'sv': 'Swedish', 'tg': 'Tajik', 'ta': 'Tamil', 'tt': 'Tatar', 'te': 'Telugu',
+                         'th': 'Thai', 'ti': 'Tigrinya', 'to': 'Tongan', 'lua': 'Tshiluba', 'tum': 'Tumbuka',
+                         'tr': 'Turkish', 'tk': 'Turkmen', 'tw': 'Twi',
+                         'ug': 'Uyghur', 'uk': 'Ukrainian', 'ur': 'Urdu', 'uz': 'Uzbek', 'vu': 'Vanuatu',
+                         'vi': 'Vietnamese', 'cy': 'Welsh', 'wo': 'Wolof', 'xh': 'Xhosa', 'yi': 'Yiddish',
+                         'yo': 'Yoruba', 'zu': 'Zulu'}
+        if language in language_dict:
+            return language_dict[language]
+        else:
+            return ""

agent/component/switch.py

@@ -54,7 +54,7 @@ class Switch(ComponentBase, ABC):
             for item in cond["items"]:
                 if not item["cpn_id"]:
                     continue
-                if item["cpn_id"].find("begin") >= 0:
+                if item["cpn_id"].lower().find("begin") >= 0 or item["cpn_id"].lower().find("answer") >= 0:
                     continue
                 cid = item["cpn_id"].split("@")[0]
                 res.append(cid)
@@ -75,7 +75,7 @@ class Switch(ComponentBase, ABC):
                             res.append(self.process_operator(p.get("value",""), item["operator"], item.get("value", "")))
                             break
                 else:
-                    out = self._canvas.get_component(cid)["obj"].output()[1]
+                    out = self._canvas.get_component(cid)["obj"].output(allow_partial=False)[1]
                     cpn_input = "" if "content" not in out.columns else " ".join([str(s) for s in out["content"]])
                     res.append(self.process_operator(cpn_input, item["operator"], item.get("value", "")))
 

agent/component/template.py

@@ -15,8 +15,11 @@
 #
 import json
 import re
+
+from jinja2 import StrictUndefined
+from jinja2.sandbox import SandboxedEnvironment
+
 from agent.component.base import ComponentBase, ComponentParamBase
-from jinja2 import Template as Jinja2Template
 
 
 class TemplateParam(ComponentParamBase):
@@ -38,36 +41,54 @@ class Template(ComponentBase):
     component_name = "Template"
 
     def get_dependent_components(self):
-        cpnts = set(
-            [
-                para["component_id"].split("@")[0]
-                for para in self._param.parameters
-                if para.get("component_id")
-                and para["component_id"].lower().find("answer") < 0
-                and para["component_id"].lower().find("begin") < 0
-            ]
-        )
+        inputs = self.get_input_elements()
+        cpnts = set([i["key"] for i in inputs if i["key"].lower().find("answer") < 0 and i["key"].lower().find("begin") < 0])
         return list(cpnts)
 
+    def get_input_elements(self):
+        key_set = set([])
+        res = []
+        for r in re.finditer(r"\{([a-z]+[:@][a-z0-9_-]+)\}", self._param.content, flags=re.IGNORECASE):
+            cpn_id = r.group(1)
+            if cpn_id in key_set:
+                continue
+            if cpn_id.lower().find("begin@") == 0:
+                cpn_id, key = cpn_id.split("@")
+                for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
+                    if p["key"] != key:
+                        continue
+                    res.append({"key": r.group(1), "name": p["name"]})
+                    key_set.add(r.group(1))
+                continue
+            cpn_nm = self._canvas.get_component_name(cpn_id)
+            if not cpn_nm:
+                continue
+            res.append({"key": cpn_id, "name": cpn_nm})
+            key_set.add(cpn_id)
+        return res
+
     def _run(self, history, **kwargs):
         content = self._param.content
 
         self._param.inputs = []
-        for para in self._param.parameters:
-            if not para.get("component_id"):
-                continue
-            component_id = para["component_id"].split("@")[0]
-            if para["component_id"].lower().find("@") >= 0:
-                cpn_id, key = para["component_id"].split("@")
+        for para in self.get_input_elements():
+            if para["key"].lower().find("begin@") == 0:
+                cpn_id, key = para["key"].split("@")
                 for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
                     if p["key"] == key:
                         value = p.get("value", "")
                         self.make_kwargs(para, kwargs, value)
+
+                        origin_pattern = "{begin@" + key + "}"
+                        new_pattern = "begin_" + key
+                        content = content.replace(origin_pattern, new_pattern)
+                        kwargs[new_pattern] = kwargs.pop(origin_pattern, "")
                         break
                 else:
                     assert False, f"Can't find parameter '{key}' for {cpn_id}"
                 continue
 
+            component_id = para["key"]
             cpn = self._canvas.get_component(component_id)["obj"]
             if cpn.component_name.lower() == "answer":
                 hist = self._canvas.get_history(1)
@@ -76,19 +97,27 @@ class Template(ComponentBase):
                 else:
                     hist = ""
                 self.make_kwargs(para, kwargs, hist)
+
+                if ":" in component_id:
+                    origin_pattern = "{" + component_id + "}"
+                    new_pattern = component_id.replace(":", "_")
+                    content = content.replace(origin_pattern, new_pattern)
+                    kwargs[new_pattern] = kwargs.pop(component_id, "")
                 continue
 
             _, out = cpn.output(allow_partial=False)
 
             result = ""
             if "content" in out.columns:
-                result = "\n".join(
-                    [o if isinstance(o, str) else str(o) for o in out["content"]]
-                )
+                result = "\n".join([o if isinstance(o, str) else str(o) for o in out["content"]])
 
             self.make_kwargs(para, kwargs, result)
 
-        template = Jinja2Template(content)
+        env = SandboxedEnvironment(
+            autoescape=True,
+            undefined=StrictUndefined,
+        )
+        template = env.from_string(content)
 
         try:
             content = template.render(kwargs)
@@ -96,26 +125,21 @@ class Template(ComponentBase):
             pass
 
         for n, v in kwargs.items():
-            try:
-                v = json.dumps(v, ensure_ascii=False)
-            except Exception:
-                pass
-            content = re.sub(
-                r"\{%s\}" % re.escape(n), v, content
-            )
-            content = re.sub(
-                r"(\\\"|\")", "", content
-            )
-            content = re.sub(
-                r"(#+)", r" \1 ", content
-            )
+            if not isinstance(v, str):
+                try:
+                    v = json.dumps(v, ensure_ascii=False)
+                except Exception:
+                    pass
+            # Process backslashes in strings, Use Lambda function to avoid escape issues
+            if isinstance(v, str):
+                v = v.replace("\\", "\\\\")
+            content = re.sub(r"\{%s\}" % re.escape(n), lambda match: v, content)
+            content = re.sub(r"(#+)", r" \1 ", content)
 
         return Template.be_output(content)
 
     def make_kwargs(self, para, kwargs, value):
-        self._param.inputs.append(
-            {"component_id": para["component_id"], "content": value}
-        )
+        self._param.inputs.append({"component_id": para["key"], "content": value})
         try:
             value = json.loads(value)
         except Exception:

agentic_reasoning/__init__.py

@@ -0,0 +1 @@
+from .deep_research import DeepResearcher as DeepResearcher

agentic_reasoning/deep_research.py

@@ -0,0 +1,236 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import logging
+import re
+from functools import partial
+from agentic_reasoning.prompts import BEGIN_SEARCH_QUERY, BEGIN_SEARCH_RESULT, END_SEARCH_RESULT, MAX_SEARCH_LIMIT, \
+    END_SEARCH_QUERY, REASON_PROMPT, RELEVANT_EXTRACTION_PROMPT
+from api.db.services.llm_service import LLMBundle
+from rag.nlp import extract_between
+from rag.prompts import kb_prompt
+from rag.utils.tavily_conn import Tavily
+
+
+class DeepResearcher:
+    def __init__(self,
+                 chat_mdl: LLMBundle,
+                 prompt_config: dict,
+                 kb_retrieve: partial = None,
+                 kg_retrieve: partial = None
+                 ):
+        self.chat_mdl = chat_mdl
+        self.prompt_config = prompt_config
+        self._kb_retrieve = kb_retrieve
+        self._kg_retrieve = kg_retrieve
+
+    def _remove_tags(text: str, start_tag: str, end_tag: str) -> str:
+        """General Tag Removal Method"""
+        pattern = re.escape(start_tag) + r"(.*?)" + re.escape(end_tag)
+        return re.sub(pattern, "", text)
+
+    @staticmethod
+    def _remove_query_tags(text: str) -> str:
+        """Remove Query Tags"""
+        return DeepResearcher._remove_tags(text, BEGIN_SEARCH_QUERY, END_SEARCH_QUERY)
+
+    @staticmethod
+    def _remove_result_tags(text: str) -> str:
+        """Remove Result Tags"""
+        return DeepResearcher._remove_tags(text, BEGIN_SEARCH_RESULT, END_SEARCH_RESULT)
+
+    def _generate_reasoning(self, msg_history):
+        """Generate reasoning steps"""
+        query_think = ""
+        if msg_history[-1]["role"] != "user":
+            msg_history.append({"role": "user", "content": "Continues reasoning with the new information.\n"})
+        else:
+            msg_history[-1]["content"] += "\n\nContinues reasoning with the new information.\n"
+            
+        for ans in self.chat_mdl.chat_streamly(REASON_PROMPT, msg_history, {"temperature": 0.7}):
+            ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+            if not ans:
+                continue
+            query_think = ans
+            yield query_think
+        return query_think
+
+    def _extract_search_queries(self, query_think, question, step_index):
+        """Extract search queries from thinking"""
+        queries = extract_between(query_think, BEGIN_SEARCH_QUERY, END_SEARCH_QUERY)
+        if not queries and step_index == 0:
+            # If this is the first step and no queries are found, use the original question as the query
+            queries = [question]
+        return queries
+
+    def _truncate_previous_reasoning(self, all_reasoning_steps):
+        """Truncate previous reasoning steps to maintain a reasonable length"""
+        truncated_prev_reasoning = ""
+        for i, step in enumerate(all_reasoning_steps):
+            truncated_prev_reasoning += f"Step {i + 1}: {step}\n\n"
+
+        prev_steps = truncated_prev_reasoning.split('\n\n')
+        if len(prev_steps) <= 5:
+            truncated_prev_reasoning = '\n\n'.join(prev_steps)
+        else:
+            truncated_prev_reasoning = ''
+            for i, step in enumerate(prev_steps):
+                if i == 0 or i >= len(prev_steps) - 4 or BEGIN_SEARCH_QUERY in step or BEGIN_SEARCH_RESULT in step:
+                    truncated_prev_reasoning += step + '\n\n'
+                else:
+                    if truncated_prev_reasoning[-len('\n\n...\n\n'):] != '\n\n...\n\n':
+                        truncated_prev_reasoning += '...\n\n'
+        
+        return truncated_prev_reasoning.strip('\n')
+
+    def _retrieve_information(self, search_query):
+        """Retrieve information from different sources"""
+        # 1. Knowledge base retrieval
+        kbinfos = []
+        try:
+            kbinfos = self._kb_retrieve(question=search_query) if self._kb_retrieve else {"chunks": [], "doc_aggs": []}
+        except Exception as e:
+            logging.error(f"Knowledge base retrieval error: {e}")
+
+        # 2. Web retrieval (if Tavily API is configured)
+        try:
+            if self.prompt_config.get("tavily_api_key"):
+                tav = Tavily(self.prompt_config["tavily_api_key"])
+                tav_res = tav.retrieve_chunks(search_query)
+                kbinfos["chunks"].extend(tav_res["chunks"])
+                kbinfos["doc_aggs"].extend(tav_res["doc_aggs"])
+        except Exception as e:
+            logging.error(f"Web retrieval error: {e}")
+
+        # 3. Knowledge graph retrieval (if configured)
+        try:
+            if self.prompt_config.get("use_kg") and self._kg_retrieve:
+                ck = self._kg_retrieve(question=search_query)
+                if ck["content_with_weight"]:
+                    kbinfos["chunks"].insert(0, ck)
+        except Exception as e:
+            logging.error(f"Knowledge graph retrieval error: {e}")
+
+        return kbinfos
+
+    def _update_chunk_info(self, chunk_info, kbinfos):
+        """Update chunk information for citations"""
+        if not chunk_info["chunks"]:
+            # If this is the first retrieval, use the retrieval results directly
+            for k in chunk_info.keys():
+                chunk_info[k] = kbinfos[k]
+        else:
+            # Merge newly retrieved information, avoiding duplicates
+            cids = [c["chunk_id"] for c in chunk_info["chunks"]]
+            for c in kbinfos["chunks"]:
+                if c["chunk_id"] not in cids:
+                    chunk_info["chunks"].append(c)
+                    
+            dids = [d["doc_id"] for d in chunk_info["doc_aggs"]]
+            for d in kbinfos["doc_aggs"]:
+                if d["doc_id"] not in dids:
+                    chunk_info["doc_aggs"].append(d)
+
+    def _extract_relevant_info(self, truncated_prev_reasoning, search_query, kbinfos):
+        """Extract and summarize relevant information"""
+        summary_think = ""
+        for ans in self.chat_mdl.chat_streamly(
+                RELEVANT_EXTRACTION_PROMPT.format(
+                    prev_reasoning=truncated_prev_reasoning,
+                    search_query=search_query,
+                    document="\n".join(kb_prompt(kbinfos, 4096))
+                ),
+                [{"role": "user",
+                  "content": f'Now you should analyze each web page and find helpful information based on the current search query "{search_query}" and previous reasoning steps.'}],
+                {"temperature": 0.7}):
+            ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+            if not ans:
+                continue
+            summary_think = ans
+            yield summary_think
+        
+        return summary_think
+
+    def thinking(self, chunk_info: dict, question: str):
+        executed_search_queries = []
+        msg_history = [{"role": "user", "content": f'Question:\"{question}\"\n'}]
+        all_reasoning_steps = []
+        think = "<think>"
+        
+        for step_index in range(MAX_SEARCH_LIMIT + 1):
+            # Check if the maximum search limit has been reached
+            if step_index == MAX_SEARCH_LIMIT - 1:
+                summary_think = f"\n{BEGIN_SEARCH_RESULT}\nThe maximum search limit is exceeded. You are not allowed to search.\n{END_SEARCH_RESULT}\n"
+                yield {"answer": think + summary_think + "</think>", "reference": {}, "audio_binary": None}
+                all_reasoning_steps.append(summary_think)
+                msg_history.append({"role": "assistant", "content": summary_think})
+                break
+
+            # Step 1: Generate reasoning
+            query_think = ""
+            for ans in self._generate_reasoning(msg_history):
+                query_think = ans
+                yield {"answer": think + self._remove_query_tags(query_think) + "</think>", "reference": {}, "audio_binary": None}
+
+            think += self._remove_query_tags(query_think)
+            all_reasoning_steps.append(query_think)
+            
+            # Step 2: Extract search queries
+            queries = self._extract_search_queries(query_think, question, step_index)
+            if not queries and step_index > 0:
+                # If not the first step and no queries, end the search process
+                break
+
+            # Process each search query
+            for search_query in queries:
+                logging.info(f"[THINK]Query: {step_index}. {search_query}")
+                msg_history.append({"role": "assistant", "content": search_query})
+                think += f"\n\n> {step_index + 1}. {search_query}\n\n"
+                yield {"answer": think + "</think>", "reference": {}, "audio_binary": None}
+
+                # Check if the query has already been executed
+                if search_query in executed_search_queries:
+                    summary_think = f"\n{BEGIN_SEARCH_RESULT}\nYou have searched this query. Please refer to previous results.\n{END_SEARCH_RESULT}\n"
+                    yield {"answer": think + summary_think + "</think>", "reference": {}, "audio_binary": None}
+                    all_reasoning_steps.append(summary_think)
+                    msg_history.append({"role": "user", "content": summary_think})
+                    think += summary_think
+                    continue
+                
+                executed_search_queries.append(search_query)
+                
+                # Step 3: Truncate previous reasoning steps
+                truncated_prev_reasoning = self._truncate_previous_reasoning(all_reasoning_steps)
+                
+                # Step 4: Retrieve information
+                kbinfos = self._retrieve_information(search_query)
+                
+                # Step 5: Update chunk information
+                self._update_chunk_info(chunk_info, kbinfos)
+                
+                # Step 6: Extract relevant information
+                think += "\n\n"
+                summary_think = ""
+                for ans in self._extract_relevant_info(truncated_prev_reasoning, search_query, kbinfos):
+                    summary_think = ans
+                    yield {"answer": think + self._remove_result_tags(summary_think) + "</think>", "reference": {}, "audio_binary": None}
+
+                all_reasoning_steps.append(summary_think)
+                msg_history.append(
+                    {"role": "user", "content": f"\n\n{BEGIN_SEARCH_RESULT}{summary_think}{END_SEARCH_RESULT}\n\n"})
+                think += self._remove_result_tags(summary_think)
+                logging.info(f"[THINK]Summary: {step_index}. {summary_think}")
+
+        yield think + "</think>"

agentic_reasoning/prompts.py

@@ -0,0 +1,113 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+BEGIN_SEARCH_QUERY = "<|begin_search_query|>"
+END_SEARCH_QUERY = "<|end_search_query|>"
+BEGIN_SEARCH_RESULT = "<|begin_search_result|>"
+END_SEARCH_RESULT = "<|end_search_result|>"
+MAX_SEARCH_LIMIT = 6
+
+REASON_PROMPT = (
+        "You are a reasoning assistant with the ability to perform dataset searches to help "
+        "you answer the user's question accurately. You have special tools:\n\n"
+        f"- To perform a search: write {BEGIN_SEARCH_QUERY} your query here {END_SEARCH_QUERY}.\n"
+        f"Then, the system will search and analyze relevant content, then provide you with helpful information in the format {BEGIN_SEARCH_RESULT} ...search results... {END_SEARCH_RESULT}.\n\n"
+        f"You can repeat the search process multiple times if necessary. The maximum number of search attempts is limited to {MAX_SEARCH_LIMIT}.\n\n"
+        "Once you have all the information you need, continue your reasoning.\n\n"
+        "-- Example 1 --\n" ########################################
+        "Question: \"Are both the directors of Jaws and Casino Royale from the same country?\"\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Who is the director of Jaws?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nThe director of Jaws is Steven Spielberg...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information.\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Where is Steven Spielberg from?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nSteven Allan Spielberg is an American filmmaker...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information...\n\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Who is the director of Casino Royale?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nCasino Royale is a 2006 spy film directed by Martin Campbell...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information...\n\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Where is Martin Campbell from?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nMartin Campbell (born 24 October 1943) is a New Zealand film and television director...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information...\n\n"
+        "Assistant:\nIt's enough to answer the question\n"
+
+        "-- Example 2 --\n" #########################################
+        "Question: \"When was the founder of craigslist born?\"\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Who was the founder of craigslist?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nCraigslist was founded by Craig Newmark...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information.\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY} When was Craig Newmark born?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nCraig Newmark was born on December 6, 1952...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information...\n\n"
+        "Assistant:\nIt's enough to answer the question\n"
+        "**Remember**:\n"
+        f"- You have a dataset to search, so you just provide a proper search query.\n"
+        f"- Use {BEGIN_SEARCH_QUERY} to request a dataset search and end with {END_SEARCH_QUERY}.\n"
+        "- The language of query MUST be as the same as 'Question' or 'search result'.\n"
+        "- If no helpful information can be found, rewrite the search query to be less and precise keywords.\n"
+        "- When done searching, continue your reasoning.\n\n"
+        'Please answer the following question. You should think step by step to solve it.\n\n'
+    )
+
+RELEVANT_EXTRACTION_PROMPT = """**Task Instruction:**
+
+    You are tasked with reading and analyzing web pages based on the following inputs: **Previous Reasoning Steps**, **Current Search Query**, and **Searched Web Pages**. Your objective is to extract relevant and helpful information for **Current Search Query** from the **Searched Web Pages** and seamlessly integrate this information into the **Previous Reasoning Steps** to continue reasoning for the original question.
+
+    **Guidelines:**
+
+    1. **Analyze the Searched Web Pages:**
+    - Carefully review the content of each searched web page.
+    - Identify factual information that is relevant to the **Current Search Query** and can aid in the reasoning process for the original question.
+
+    2. **Extract Relevant Information:**
+    - Select the information from the Searched Web Pages that directly contributes to advancing the **Previous Reasoning Steps**.
+    - Ensure that the extracted information is accurate and relevant.
+
+    3. **Output Format:**
+    - **If the web pages provide helpful information for current search query:** Present the information beginning with `**Final Information**` as shown below.
+    - The language of query **MUST BE** as the same as 'Search Query' or 'Web Pages'.\n"
+    **Final Information**
+
+    [Helpful information]
+
+    - **If the web pages do not provide any helpful information for current search query:** Output the following text.
+
+    **Final Information**
+
+    No helpful information found.
+
+    **Inputs:**
+    - **Previous Reasoning Steps:**  
+    {prev_reasoning}
+
+    - **Current Search Query:**  
+    {search_query}
+
+    - **Searched Web Pages:**  
+    {document}
+
+    """

api/apps/__init__.py

@@ -83,7 +83,7 @@ app.errorhandler(Exception)(server_error_response)
 app.config["SESSION_PERMANENT"] = False
 app.config["SESSION_TYPE"] = "filesystem"
 app.config["MAX_CONTENT_LENGTH"] = int(
-    os.environ.get("MAX_CONTENT_LENGTH", 128 * 1024 * 1024)
+    os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024 * 1024)
 )
 
 Session(app)
@@ -107,7 +107,7 @@ def search_pages_path(pages_dir):
 def register_page(page_path):
     path = f"{page_path}"
 
-    page_name = page_path.stem.rstrip("_app")
+    page_name = page_path.stem.removesuffix("_app")
     module_name = ".".join(
         page_path.parts[page_path.parts.index("api"): -1] + (page_name,)
     )
@@ -119,8 +119,9 @@ def register_page(page_path):
     sys.modules[module_name] = page
     spec.loader.exec_module(page)
     page_name = getattr(page, "page_name", page_name)
+    sdk_path = "\\sdk\\" if sys.platform.startswith("win") else "/sdk/"
     url_prefix = (
-        f"/api/{API_VERSION}" if "/sdk/" in path else f"/{API_VERSION}/{page_name}"
+        f"/api/{API_VERSION}" if sdk_path in path else f"/{API_VERSION}/{page_name}"
     )
 
     app.register_blueprint(page.manager, url_prefix=url_prefix)
@@ -145,10 +146,23 @@ def load_user(web_request):
     if authorization:
         try:
             access_token = str(jwt.loads(authorization))
+            
+            if not access_token or not access_token.strip():
+                logging.warning("Authentication attempt with empty access token")
+                return None
+            
+            # Access tokens should be UUIDs (32 hex characters)
+            if len(access_token.strip()) < 32:
+                logging.warning(f"Authentication attempt with invalid token format: {len(access_token)} chars")
+                return None
+            
             user = UserService.query(
                 access_token=access_token, status=StatusEnum.VALID.value
             )
             if user:
+                if not user[0].access_token or not user[0].access_token.strip():
+                    logging.warning(f"User {user[0].email} has empty access_token in database")
+                    return None
                 return user[0]
             else:
                 return None

api/apps/api_app.py

@@ -18,14 +18,14 @@ import os
 import re
 from datetime import datetime, timedelta
 from flask import request, Response
-from api.db.services.llm_service import TenantLLMService
+from api.db.services.llm_service import LLMBundle
 from flask_login import login_required, current_user
 
-from api.db import FileType, LLMType, ParserType, FileSource
+from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileType, LLMType, ParserType, FileSource
 from api.db.db_models import APIToken, Task, File
 from api.db.services import duplicate_name
 from api.db.services.api_service import APITokenService, API4ConversationService
-from api.db.services.dialog_service import DialogService, chat, keyword_extraction, label_question
+from api.db.services.dialog_service import DialogService, chat
 from api.db.services.document_service import DocumentService, doc_upload_and_parse
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
@@ -38,6 +38,8 @@ from api.utils.api_utils import server_error_response, get_data_error_result, ge
     generate_confirmation_token
 
 from api.utils.file_utils import filename_type, thumbnail
+from rag.app.tag import label_question
+from rag.prompts import keyword_extraction
 from rag.utils.storage_factory import STORAGE_IMPL
 
 from api.db.services.canvas_service import UserCanvasService
@@ -343,7 +345,7 @@ def completion():
 
 @manager.route('/conversation/<conversation_id>', methods=['GET'])  # noqa: F821
 # @login_required
-def get(conversation_id):
+def get_conversation(conversation_id):
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
@@ -477,7 +479,7 @@ def upload():
                 doc = doc.to_dict()
                 doc["tenant_id"] = tenant_id
                 bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
-                queue_tasks(doc, bucket, name)
+                queue_tasks(doc, bucket, name, 0)
             except Exception as e:
                 return server_error_response(e)
 
@@ -546,6 +548,31 @@ def list_chunks():
 
     return get_json_result(data=res)
 
+@manager.route('/get_chunk/<chunk_id>', methods=['GET'])  # noqa: F821
+# @login_required
+def get_chunk(chunk_id):
+    from rag.nlp import search
+    token = request.headers.get('Authorization').split()[1]
+    objs = APIToken.query(token=token)
+    if not objs:
+        return get_json_result(
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+    try:
+        tenant_id = objs[0].tenant_id
+        kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
+        chunk = settings.docStoreConn.get(chunk_id, search.index_name(tenant_id), kb_ids)
+        if chunk is None:
+            return server_error_response(Exception("Chunk not found"))
+        k = []
+        for n in chunk.keys():
+            if re.search(r"(_vec$|_sm_|_tks|_ltks)", n):
+                k.append(n)
+        for n in k:
+            del chunk[n]
+
+        return get_json_result(data=chunk)
+    except Exception as e:
+        return server_error_response(e)
 
 @manager.route('/list_kb_docs', methods=['POST'])  # noqa: F821
 # @login_required
@@ -575,10 +602,23 @@ def list_kb_docs():
     orderby = req.get("orderby", "create_time")
     desc = req.get("desc", True)
     keywords = req.get("keywords", "")
-
+    status = req.get("status", [])
+    if status:
+        invalid_status = {s for s in status if s not in VALID_TASK_STATUS}
+        if invalid_status:
+            return get_data_error_result(
+                message=f"Invalid filter status conditions: {', '.join(invalid_status)}"
+            )
+    types = req.get("types", [])
+    if types:
+        invalid_types = {t for t in types if t not in VALID_FILE_TYPES}
+        if invalid_types:
+            return get_data_error_result(
+                message=f"Invalid filter conditions: {', '.join(invalid_types)} type{'s' if len(invalid_types) > 1 else ''}"
+            )
     try:
         docs, tol = DocumentService.get_by_kb_id(
-            kb_id, page_number, items_per_page, orderby, desc, keywords)
+            kb_id, page_number, items_per_page, orderby, desc, keywords, status, types)
         docs = [{"doc_id": doc['id'], "doc_name": doc['name']} for doc in docs]
 
         return get_json_result(data={"total": tol, "docs": docs})
@@ -613,7 +653,7 @@ def document_rm():
     tenant_id = objs[0].tenant_id
     req = request.json
     try:
-        doc_ids = [DocumentService.get_doc_id_by_doc_name(doc_name) for doc_name in req.get("doc_names", [])]
+        doc_ids = DocumentService.get_doc_ids_by_doc_names(req.get("doc_names", []))
         for doc_id in req.get("doc_ids", []):
             if doc_id not in doc_ids:
                 doc_ids.append(doc_id)
@@ -631,11 +671,16 @@ def document_rm():
     FileService.init_knowledgebase_docs(pf_id, tenant_id)
 
     errors = ""
+    docs = DocumentService.get_by_ids(doc_ids)
+    doc_dic = {}
+    for doc in docs:
+        doc_dic[doc.id] = doc
+
     for doc_id in doc_ids:
         try:
-            e, doc = DocumentService.get_by_id(doc_id)
-            if not e:
+            if doc_id not in doc_dic:
                 return get_data_error_result(message="Document not found!")
+            doc = doc_dic[doc_id]
             tenant_id = DocumentService.get_tenant_id(doc_id)
             if not tenant_id:
                 return get_data_error_result(message="Tenant not found!")
@@ -816,10 +861,11 @@ def retrieval():
     doc_ids = req.get("doc_ids", [])
     question = req.get("question")
     page = int(req.get("page", 1))
-    size = int(req.get("size", 30))
+    size = int(req.get("page_size", 30))
     similarity_threshold = float(req.get("similarity_threshold", 0.2))
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
     top = int(req.get("top_k", 1024))
+    highlight = bool(req.get("highlight", False)) 
 
     try:
         kbs = KnowledgebaseService.get_by_ids(kb_ids)
@@ -829,18 +875,16 @@ def retrieval():
                 data=False, message='Knowledge bases use different embedding models or does not exist."',
                 code=settings.RetCode.AUTHENTICATION_ERROR)
 
-        embd_mdl = TenantLLMService.model_instance(
-            kbs[0].tenant_id, LLMType.EMBEDDING.value, llm_name=kbs[0].embd_id)
+        embd_mdl = LLMBundle(kbs[0].tenant_id, LLMType.EMBEDDING, llm_name=kbs[0].embd_id)
         rerank_mdl = None
         if req.get("rerank_id"):
-            rerank_mdl = TenantLLMService.model_instance(
-                kbs[0].tenant_id, LLMType.RERANK.value, llm_name=req["rerank_id"])
+            rerank_mdl = LLMBundle(kbs[0].tenant_id, LLMType.RERANK, llm_name=req["rerank_id"])
         if req.get("keyword", False):
-            chat_mdl = TenantLLMService.model_instance(kbs[0].tenant_id, LLMType.CHAT)
+            chat_mdl = LLMBundle(kbs[0].tenant_id, LLMType.CHAT)
             question += keyword_extraction(chat_mdl, question)
         ranks = settings.retrievaler.retrieval(question, embd_mdl, kbs[0].tenant_id, kb_ids, page, size,
                                                similarity_threshold, vector_similarity_weight, top,
-                                               doc_ids, rerank_mdl=rerank_mdl,
+                                               doc_ids, rerank_mdl=rerank_mdl, highlight= highlight,
                                                rank_feature=label_question(question, kbs))
         for c in ranks["chunks"]:
             c.pop("vector", None)

api/apps/auth/README.md

@@ -0,0 +1,76 @@
+# Auth
+
+The Auth module provides implementations of OAuth2 and OpenID Connect (OIDC) authentication for integration with third-party identity providers. 
+
+**Features**
+
+- Supports both OAuth2 and OIDC authentication protocols
+- Automatic OIDC configuration discovery (via `/.well-known/openid-configuration`)
+- JWT token validation
+- Unified user information handling
+
+## Usage
+
+```python
+# OAuth2 configuration
+oauth_config = {
+    "type": "oauth2",
+    "client_id": "your_client_id",
+    "client_secret": "your_client_secret",
+    "authorization_url": "https://your-oauth-provider.com/oauth/authorize",
+    "token_url": "https://your-oauth-provider.com/oauth/token",
+    "userinfo_url": "https://your-oauth-provider.com/oauth/userinfo",
+    "redirect_uri": "https://your-app.com/v1/user/oauth/callback/<channel>"
+}
+
+# OIDC configuration
+oidc_config = {
+    "type": "oidc",
+    "issuer": "https://your-oauth-provider.com/oidc",
+    "client_id": "your_client_id",
+    "client_secret": "your_client_secret",
+    "redirect_uri": "https://your-app.com/v1/user/oauth/callback/<channel>"
+}
+
+# Github OAuth configuration
+github_config = {
+    "type": "github"
+    "client_id": "your_client_id",
+    "client_secret": "your_client_secret",
+    "redirect_uri": "https://your-app.com/v1/user/oauth/callback/<channel>"
+}
+
+# Get client instance
+client = get_auth_client(oauth_config)
+```
+
+### Authentication Flow
+
+1. Get authorization URL:
+```python
+auth_url = client.get_authorization_url()
+```
+
+2. After user authorization, exchange authorization code for token:
+```python
+token_response = client.exchange_code_for_token(authorization_code)
+access_token = token_response["access_token"]
+```
+
+3. Fetch user information:
+```python
+user_info = client.fetch_user_info(access_token)
+```
+
+## User Information Structure
+
+All authentication methods return user information following this structure:
+
+```python
+{
+    "email": "user@example.com",
+    "username": "username",
+    "nickname": "User Name",
+    "avatar_url": "https://example.com/avatar.jpg"
+}
+```

api/apps/auth/__init__.py

@@ -0,0 +1,40 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from .oauth import OAuthClient
+from .oidc import OIDCClient
+from .github import GithubOAuthClient
+
+
+CLIENT_TYPES = {
+    "oauth2": OAuthClient,
+    "oidc": OIDCClient,
+    "github": GithubOAuthClient
+}
+
+
+def get_auth_client(config)->OAuthClient:
+    channel_type = str(config.get("type", "")).lower()
+    if channel_type == "":
+        if config.get("issuer"):
+            channel_type = "oidc"
+        else:
+            channel_type = "oauth2"
+    client_class = CLIENT_TYPES.get(channel_type)
+    if not client_class:
+        raise ValueError(f"Unsupported type: {channel_type}")
+
+    return client_class(config)

api/apps/auth/github.py

@@ -0,0 +1,63 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import requests
+from .oauth import OAuthClient, UserInfo
+
+
+class GithubOAuthClient(OAuthClient):
+    def __init__(self, config):
+        """
+        Initialize the GithubOAuthClient with the provider's configuration.
+        """
+        config.update({
+            "authorization_url": "https://github.com/login/oauth/authorize",
+            "token_url": "https://github.com/login/oauth/access_token",
+            "userinfo_url": "https://api.github.com/user",
+            "scope": "user:email"
+        })
+        super().__init__(config)
+
+
+    def fetch_user_info(self, access_token, **kwargs):
+        """
+        Fetch github user info.
+        """
+        user_info = {}
+        try:
+            headers = {"Authorization": f"Bearer {access_token}"}
+            # user info
+            response = requests.get(self.userinfo_url, headers=headers, timeout=self.http_request_timeout)
+            response.raise_for_status()
+            user_info.update(response.json())
+            # email info
+            response = requests.get(self.userinfo_url+"/emails", headers=headers, timeout=self.http_request_timeout)
+            response.raise_for_status()
+            email_info = response.json()
+            user_info["email"] = next(
+                (email for email in email_info if email["primary"]), None
+            )["email"]
+            return self.normalize_user_info(user_info)
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to fetch github user info: {e}")
+
+
+    def normalize_user_info(self, user_info):
+        email = user_info.get("email")
+        username = user_info.get("login", str(email).split("@")[0])
+        nickname = user_info.get("name", username)
+        avatar_url = user_info.get("avatar_url", "")
+        return UserInfo(email=email, username=username, nickname=nickname, avatar_url=avatar_url)

api/apps/auth/oauth.py

@@ -0,0 +1,110 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import requests
+import urllib.parse
+
+
+class UserInfo:
+    def __init__(self, email, username, nickname, avatar_url):
+        self.email = email
+        self.username = username
+        self.nickname = nickname
+        self.avatar_url = avatar_url
+    
+    def to_dict(self):
+        return {key: value for key, value in self.__dict__.items()}
+
+
+class OAuthClient:
+    def __init__(self, config):
+        """
+        Initialize the OAuthClient with the provider's configuration.
+        """
+        self.client_id = config["client_id"]
+        self.client_secret = config["client_secret"]
+        self.authorization_url = config["authorization_url"]
+        self.token_url = config["token_url"]
+        self.userinfo_url = config["userinfo_url"]
+        self.redirect_uri = config["redirect_uri"]
+        self.scope = config.get("scope", None)
+
+        self.http_request_timeout = 7
+
+
+    def get_authorization_url(self, state=None):
+        """
+        Generate the authorization URL for user login.
+        """
+        params = {
+            "client_id": self.client_id,
+            "redirect_uri": self.redirect_uri,
+            "response_type": "code",
+        }
+        if self.scope:
+            params["scope"] = self.scope
+        if state:
+            params["state"] = state
+        authorization_url = f"{self.authorization_url}?{urllib.parse.urlencode(params)}"
+        return authorization_url
+
+
+    def exchange_code_for_token(self, code):
+        """
+        Exchange authorization code for access token.
+        """
+        try:
+            payload = {
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "code": code,
+                "redirect_uri": self.redirect_uri,
+                "grant_type": "authorization_code"
+            }
+            response = requests.post(
+                self.token_url,
+                data=payload,
+                headers={"Accept": "application/json"},
+                timeout=self.http_request_timeout
+            )
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to exchange authorization code for token: {e}")
+
+
+    def fetch_user_info(self, access_token, **kwargs):
+        """
+        Fetch user information using access token.
+        """
+        try:
+            headers = {"Authorization": f"Bearer {access_token}"}
+            response = requests.get(self.userinfo_url, headers=headers, timeout=self.http_request_timeout)
+            response.raise_for_status()
+            user_info = response.json()
+            return self.normalize_user_info(user_info)
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to fetch user info: {e}")
+
+
+    def normalize_user_info(self, user_info):
+        email = user_info.get("email")
+        username = user_info.get("username", str(email).split("@")[0])
+        nickname = user_info.get("nickname", username)
+        avatar_url = user_info.get("avatar_url", None)
+        if avatar_url is None:
+            avatar_url = user_info.get("picture", "")
+        return UserInfo(email=email, username=username, nickname=nickname, avatar_url=avatar_url)

api/apps/auth/oidc.py

@@ -0,0 +1,99 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import jwt
+import requests
+from .oauth import OAuthClient
+
+
+class OIDCClient(OAuthClient):
+    def __init__(self, config):
+        """
+        Initialize the OIDCClient with the provider's configuration.
+        Use `issuer` as the single source of truth for configuration discovery.
+        """
+        self.issuer = config.get("issuer")
+        if not self.issuer:
+            raise ValueError("Missing issuer in configuration.")
+
+        oidc_metadata = self._load_oidc_metadata(self.issuer)
+        config.update({
+            'issuer': oidc_metadata['issuer'],
+            'jwks_uri': oidc_metadata['jwks_uri'], 
+            'authorization_url': oidc_metadata['authorization_endpoint'],
+            'token_url': oidc_metadata['token_endpoint'],
+            'userinfo_url': oidc_metadata['userinfo_endpoint']
+        })
+
+        super().__init__(config)
+        self.issuer = config['issuer']
+        self.jwks_uri = config['jwks_uri']
+
+
+    def _load_oidc_metadata(self, issuer):
+        """
+        Load OIDC metadata from `/.well-known/openid-configuration`.
+        """
+        try:
+            metadata_url = f"{issuer}/.well-known/openid-configuration"
+            response = requests.get(metadata_url, timeout=7)
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to fetch OIDC metadata: {e}")
+
+
+    def parse_id_token(self, id_token):
+        """
+        Parse and validate OIDC ID Token (JWT format) with signature verification.
+        """
+        try:
+            # Decode JWT header without verifying signature
+            headers = jwt.get_unverified_header(id_token)
+            
+            # OIDC usually uses `RS256` for signing
+            alg = headers.get("alg", "RS256")
+
+            # Use PyJWT's PyJWKClient to fetch JWKS and find signing key
+            jwks_cli = jwt.PyJWKClient(self.jwks_uri)
+            signing_key = jwks_cli.get_signing_key_from_jwt(id_token).key
+
+            # Decode and verify signature
+            decoded_token = jwt.decode(
+                id_token,
+                key=signing_key,
+                algorithms=[alg],  
+                audience=str(self.client_id),
+                issuer=self.issuer,
+            )
+            return decoded_token
+        except Exception as e:
+            raise ValueError(f"Error parsing ID Token: {e}")
+
+
+    def fetch_user_info(self, access_token, id_token=None, **kwargs):
+        """
+        Fetch user info.
+        """
+        user_info = {}
+        if id_token:
+            user_info = self.parse_id_token(id_token)
+        user_info.update(super().fetch_user_info(access_token).to_dict())
+        return self.normalize_user_info(user_info)
+
+
+    def normalize_user_info(self, user_info):
+        return super().normalize_user_info(user_info)

api/apps/canvas_app.py

@@ -18,13 +18,15 @@ import traceback
 from flask import request, Response
 from flask_login import login_required, current_user
 from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService
+from api.db.services.user_service import TenantService
+from api.db.services.user_canvas_version import UserCanvasVersionService
 from api.settings import RetCode
 from api.utils import get_uuid
 from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result
 from agent.canvas import Canvas
 from peewee import MySQLDatabase, PostgresqlDatabase
 from api.db.db_models import APIToken
-
+import time
 
 @manager.route('/templates', methods=['GET'])  # noqa: F821
 @login_required
@@ -61,7 +63,6 @@ def save():
     req["user_id"] = current_user.id
     if not isinstance(req["dsl"], str):
         req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
-
     req["dsl"] = json.loads(req["dsl"])
     if "id" not in req:
         if UserCanvasService.query(user_id=current_user.id, title=req["title"].strip()):
@@ -75,16 +76,21 @@ def save():
                 data=False, message='Only owner of canvas authorized for this operation.',
                 code=RetCode.OPERATING_ERROR)
         UserCanvasService.update_by_id(req["id"], req)
+    # save version    
+    UserCanvasVersionService.insert( user_canvas_id=req["id"], dsl=req["dsl"], title="{0}_{1}".format(req["title"], time.strftime("%Y_%m_%d_%H_%M_%S")))
+    UserCanvasVersionService.delete_all_versions(req["id"])
     return get_json_result(data=req)
 
+ 
+
 
 @manager.route('/get/<canvas_id>', methods=['GET'])  # noqa: F821
 @login_required
 def get(canvas_id):
-    e, c = UserCanvasService.get_by_id(canvas_id)
+    e, c = UserCanvasService.get_by_tenant_id(canvas_id)
     if not e:
         return get_data_error_result(message="canvas not found.")
-    return get_json_result(data=c.to_dict())
+    return get_json_result(data=c)
 
 @manager.route('/getsse/<canvas_id>', methods=['GET'])  # type: ignore # noqa: F821
 def getsse(canvas_id):
@@ -107,6 +113,7 @@ def getsse(canvas_id):
 def run():
     req = request.json
     stream = req.get("stream", True)
+    running_hint_text = req.get("running_hint_text", "")
     e, cvs = UserCanvasService.get_by_id(req["id"])
     if not e:
         return get_data_error_result(message="canvas not found.")
@@ -132,7 +139,7 @@ def run():
         def sse():
             nonlocal answer, cvs
             try:
-                for ans in canvas.run(stream=True):
+                for ans in canvas.run(running_hint_text = running_hint_text, stream=True):
                     if ans.get("running_status"):
                         yield "data:" + json.dumps({"code": 0, "message": "",
                                                     "data": {"answer": ans["content"],
@@ -170,7 +177,7 @@ def run():
         resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
         return resp
 
-    for answer in canvas.run(stream=False):
+    for answer in canvas.run(running_hint_text = running_hint_text, stream=False):
         if answer.get("running_status"):
             continue
         final_ans["content"] = "\n".join(answer["content"]) if "content" in answer else ""
@@ -242,7 +249,9 @@ def debug():
                 code=RetCode.OPERATING_ERROR)
 
         canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id)
-        canvas.get_component(req["component_id"])["obj"]._param.debug_inputs = req["params"]
+        componant = canvas.get_component(req["component_id"])["obj"]
+        componant.reset()
+        componant._param.debug_inputs = req["params"]
         df = canvas.get_component(req["component_id"])["obj"].debug()
         return get_json_result(data=df.to_dict(orient="records"))
     except Exception as e:
@@ -283,4 +292,62 @@ def test_db_connect():
         return get_json_result(data="Database Connection Successful!")
     except Exception as e:
         return server_error_response(e)
-
+#api get list version dsl of canvas
+@manager.route('/getlistversion/<canvas_id>', methods=['GET'])  # noqa: F821
+@login_required
+def getlistversion(canvas_id):
+    try:
+        list =sorted([c.to_dict() for c in UserCanvasVersionService.list_by_canvas_id(canvas_id)], key=lambda x: x["update_time"]*-1)
+        return get_json_result(data=list)
+    except Exception as e:
+        return get_data_error_result(message=f"Error getting history files: {e}")
+#api get version dsl of canvas
+@manager.route('/getversion/<version_id>', methods=['GET'])  # noqa: F821
+@login_required
+def getversion( version_id):
+    try:
+      
+        e, version = UserCanvasVersionService.get_by_id(version_id)
+        if version:
+            return get_json_result(data=version.to_dict())
+    except Exception as e:
+        return get_json_result(data=f"Error getting history file: {e}")
+@manager.route('/listteam', methods=['GET'])  # noqa: F821
+@login_required
+def list_kbs():
+    keywords = request.args.get("keywords", "")
+    page_number = int(request.args.get("page", 1))
+    items_per_page = int(request.args.get("page_size", 150))
+    orderby = request.args.get("orderby", "create_time")
+    desc = request.args.get("desc", True)
+    try:
+        tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
+        kbs, total = UserCanvasService.get_by_tenant_ids(
+            [m["tenant_id"] for m in tenants], current_user.id, page_number,
+            items_per_page, orderby, desc, keywords)
+        return get_json_result(data={"kbs": kbs, "total": total})
+    except Exception as e:
+        return server_error_response(e)
+@manager.route('/setting', methods=['POST'])  # noqa: F821
+@validate_request("id", "title", "permission")
+@login_required
+def setting():
+    req = request.json
+    req["user_id"] = current_user.id
+    e,flow = UserCanvasService.get_by_id(req["id"])
+    if not e:
+        return get_data_error_result(message="canvas not found.")
+    flow = flow.to_dict()
+    flow["title"] = req["title"]
+    if req["description"]:
+        flow["description"] = req["description"]
+    if req["permission"]:
+        flow["permission"] = req["permission"]
+    if req["avatar"]:
+        flow["avatar"] = req["avatar"]
+    if not UserCanvasService.query(user_id=current_user.id, id=req["id"]):
+        return get_json_result(
+            data=False, message='Only owner of canvas authorized for this operation.',
+            code=RetCode.OPERATING_ERROR)
+    num= UserCanvasService.update_by_id(req["id"], flow)
+    return get_json_result(data=num)

api/apps/chunk_app.py

@@ -19,9 +19,10 @@ import json
 from flask import request
 from flask_login import login_required, current_user
 
-from api.db.services.dialog_service import keyword_extraction, label_question
 from rag.app.qa import rmPrefix, beAdoc
+from rag.app.tag import label_question
 from rag.nlp import search, rag_tokenizer
+from rag.prompts import keyword_extraction, cross_languages
 from rag.settings import PAGERANK_FLD
 from rag.utils import rmSpace
 from api.db import LLMType, ParserType
@@ -36,6 +37,7 @@ import xxhash
 import re
 
 
+
 @manager.route('/list', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("doc_id")
@@ -93,12 +95,14 @@ def get():
         tenants = UserTenantService.query(user_id=current_user.id)
         if not tenants:
             return get_data_error_result(message="Tenant not found!")
-        tenant_id = tenants[0].tenant_id
-
-        kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
-        chunk = settings.docStoreConn.get(chunk_id, search.index_name(tenant_id), kb_ids)
+        for tenant in tenants:
+            kb_ids = KnowledgebaseService.get_kb_ids(tenant.tenant_id)
+            chunk = settings.docStoreConn.get(chunk_id, search.index_name(tenant.tenant_id), kb_ids)
+            if chunk:
+                break
         if chunk is None:
             return server_error_response(Exception("Chunk not found"))
+
         k = []
         for n in chunk.keys():
             if re.search(r"(_vec$|_sm_|_tks|_ltks)", n):
@@ -191,6 +195,7 @@ def switch():
 @login_required
 @validate_request("chunk_ids", "doc_id")
 def rm():
+    from rag.utils.storage_factory import STORAGE_IMPL
     req = request.json
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
@@ -201,6 +206,9 @@ def rm():
         deleted_chunk_ids = req["chunk_ids"]
         chunk_number = len(deleted_chunk_ids)
         DocumentService.decrement_chunk_num(doc.id, doc.kb_id, 1, chunk_number, 0)
+        for cid in deleted_chunk_ids:
+            if STORAGE_IMPL.obj_exist(doc.kb_id, cid):
+                STORAGE_IMPL.rm(doc.kb_id, cid)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
@@ -272,6 +280,7 @@ def retrieval_test():
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
     use_kg = req.get("use_kg", False)
     top = int(req.get("top_k", 1024))
+    langs = req.get("cross_languages", [])
     tenant_ids = []
 
     try:
@@ -291,6 +300,9 @@ def retrieval_test():
         if not e:
             return get_data_error_result(message="Knowledgebase not found!")
 
+        if langs:
+            question = cross_languages(kb.tenant_id, None, question, langs)
+
         embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
 
         rerank_mdl = None

api/apps/conversation_app.py

@@ -17,29 +17,36 @@ import json
 import re
 import traceback
 from copy import deepcopy
-from api.db.db_models import APIToken
 
-from api.db.services.conversation_service import ConversationService, structure_answer
-from api.db.services.user_service import UserTenantService
-from flask import request, Response
-from flask_login import login_required, current_user
+import trio
+from flask import Response, request
+from flask_login import current_user, login_required
 
+from api import settings
 from api.db import LLMType
-from api.db.services.dialog_service import DialogService, chat, ask, label_question
+from api.db.db_models import APIToken
+from api.db.services.conversation_service import ConversationService, structure_answer
+from api.db.services.dialog_service import DialogService, ask, chat
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle, TenantService
-from api import settings
-from api.utils.api_utils import get_json_result
-from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
+from api.db.services.user_service import UserTenantService
+from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request
 from graphrag.general.mind_map_extractor import MindMapExtractor
+from rag.app.tag import label_question
 
 
-@manager.route('/set', methods=['POST'])  # noqa: F821
+@manager.route("/set", methods=["POST"])  # noqa: F821
 @login_required
 def set_conversation():
     req = request.json
     conv_id = req.get("conversation_id")
     is_new = req.get("is_new")
+    name = req.get("name", "New conversation")
+    req["user_id"] = current_user.id
+
+    if len(name) > 255:
+        name = name[0:255]
+
     del req["is_new"]
     if not is_new:
         del req["conversation_id"]
@@ -48,8 +55,7 @@ def set_conversation():
                 return get_data_error_result(message="Conversation not found!")
             e, conv = ConversationService.get_by_id(conv_id)
             if not e:
-                return get_data_error_result(
-                    message="Fail to update a conversation!")
+                return get_data_error_result(message="Fail to update a conversation!")
             conv = conv.to_dict()
             return get_json_result(data=conv)
         except Exception as e:
@@ -59,38 +65,30 @@ def set_conversation():
         e, dia = DialogService.get_by_id(req["dialog_id"])
         if not e:
             return get_data_error_result(message="Dialog not found")
-        conv = {
-            "id": conv_id,
-            "dialog_id": req["dialog_id"],
-            "name": req.get("name", "New conversation"),
-            "message": [{"role": "assistant", "content": dia.prompt_config["prologue"]}]
-        }
+        conv = {"id": conv_id, "dialog_id": req["dialog_id"], "name": name, "message": [{"role": "assistant", "content": dia.prompt_config["prologue"]}],"user_id": current_user.id}
         ConversationService.save(**conv)
         return get_json_result(data=conv)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/get', methods=['GET'])  # noqa: F821
+@manager.route("/get", methods=["GET"])  # noqa: F821
 @login_required
 def get():
     conv_id = request.args["conversation_id"]
     try:
-        
         e, conv = ConversationService.get_by_id(conv_id)
         if not e:
             return get_data_error_result(message="Conversation not found!")
         tenants = UserTenantService.query(user_id=current_user.id)
-        avatar =None
+        avatar = None
         for tenant in tenants:
             dialog = DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id)
-            if dialog and len(dialog)>0:
+            if dialog and len(dialog) > 0:
                 avatar = dialog[0].icon
                 break
         else:
-            return get_json_result(
-                data=False, message='Only owner of conversation authorized for this operation.',
-                code=settings.RetCode.OPERATING_ERROR)
+            return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
 
         def get_value(d, k1, k2):
             return d.get(k1, d.get(k2))
@@ -98,26 +96,30 @@ def get():
         for ref in conv.reference:
             if isinstance(ref, list):
                 continue
-            ref["chunks"] = [{
-                "id": get_value(ck, "chunk_id", "id"),
-                "content": get_value(ck, "content", "content_with_weight"),
-                "document_id": get_value(ck, "doc_id", "document_id"),
-                "document_name": get_value(ck, "docnm_kwd", "document_name"),
-                "dataset_id": get_value(ck, "kb_id", "dataset_id"),
-                "image_id": get_value(ck, "image_id", "img_id"),
-                "positions": get_value(ck, "positions", "position_int"),
-            } for ck in ref.get("chunks", [])]
+            ref["chunks"] = [
+                {
+                    "id": get_value(ck, "chunk_id", "id"),
+                    "content": get_value(ck, "content", "content_with_weight"),
+                    "document_id": get_value(ck, "doc_id", "document_id"),
+                    "document_name": get_value(ck, "docnm_kwd", "document_name"),
+                    "dataset_id": get_value(ck, "kb_id", "dataset_id"),
+                    "image_id": get_value(ck, "image_id", "img_id"),
+                    "positions": get_value(ck, "positions", "position_int"),
+                    "doc_type": get_value(ck, "doc_type", "doc_type_kwd"),
+                }
+                for ck in ref.get("chunks", [])
+            ]
 
         conv = conv.to_dict()
-        conv["avatar"]=avatar
+        conv["avatar"] = avatar
         return get_json_result(data=conv)
     except Exception as e:
         return server_error_response(e)
 
-@manager.route('/getsse/<dialog_id>', methods=['GET'])  # type: ignore # noqa: F821
+
+@manager.route("/getsse/<dialog_id>", methods=["GET"])  # type: ignore # noqa: F821
 def getsse(dialog_id):
-    
-    token = request.headers.get('Authorization').split()
+    token = request.headers.get("Authorization").split()
     if len(token) != 2:
         return get_data_error_result(message='Authorization is not valid!"')
     token = token[1]
@@ -129,13 +131,14 @@ def getsse(dialog_id):
         if not e:
             return get_data_error_result(message="Dialog not found!")
         conv = conv.to_dict()
-        conv["avatar"]= conv["icon"]
+        conv["avatar"] = conv["icon"]
         del conv["icon"]
         return get_json_result(data=conv)
     except Exception as e:
         return server_error_response(e)
 
-@manager.route('/rm', methods=['POST'])  # noqa: F821
+
+@manager.route("/rm", methods=["POST"])  # noqa: F821
 @login_required
 def rm():
     conv_ids = request.json["conversation_ids"]
@@ -149,28 +152,21 @@ def rm():
                 if DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id):
                     break
             else:
-                return get_json_result(
-                    data=False, message='Only owner of conversation authorized for this operation.',
-                    code=settings.RetCode.OPERATING_ERROR)
+                return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
             ConversationService.delete_by_id(cid)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/list', methods=['GET'])  # noqa: F821
+@manager.route("/list", methods=["GET"])  # noqa: F821
 @login_required
 def list_convsersation():
     dialog_id = request.args["dialog_id"]
     try:
         if not DialogService.query(tenant_id=current_user.id, id=dialog_id):
-            return get_json_result(
-                data=False, message='Only owner of dialog authorized for this operation.',
-                code=settings.RetCode.OPERATING_ERROR)
-        convs = ConversationService.query(
-            dialog_id=dialog_id,
-            order_by=ConversationService.model.create_time,
-            reverse=True)
+            return get_json_result(data=False, message="Only owner of dialog authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+        convs = ConversationService.query(dialog_id=dialog_id, order_by=ConversationService.model.create_time, reverse=True)
 
         convs = [d.to_dict() for d in convs]
         return get_json_result(data=convs)
@@ -178,7 +174,7 @@ def list_convsersation():
         return server_error_response(e)
 
 
-@manager.route('/completion', methods=['POST'])  # noqa: F821
+@manager.route("/completion", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("conversation_id", "messages")
 def completion():
@@ -205,25 +201,31 @@ def completion():
         if not conv.reference:
             conv.reference = []
         else:
+
             def get_value(d, k1, k2):
                 return d.get(k1, d.get(k2))
 
             for ref in conv.reference:
                 if isinstance(ref, list):
                     continue
-                ref["chunks"] = [{
-                    "id": get_value(ck, "chunk_id", "id"),
-                    "content": get_value(ck, "content", "content_with_weight"),
-                    "document_id": get_value(ck, "doc_id", "document_id"),
-                    "document_name": get_value(ck, "docnm_kwd", "document_name"),
-                    "dataset_id": get_value(ck, "kb_id", "dataset_id"),
-                    "image_id": get_value(ck, "image_id", "img_id"),
-                    "positions": get_value(ck, "positions", "position_int"),
-                } for ck in ref.get("chunks", [])]
+                ref["chunks"] = [
+                    {
+                        "id": get_value(ck, "chunk_id", "id"),
+                        "content": get_value(ck, "content", "content_with_weight"),
+                        "document_id": get_value(ck, "doc_id", "document_id"),
+                        "document_name": get_value(ck, "docnm_kwd", "document_name"),
+                        "dataset_id": get_value(ck, "kb_id", "dataset_id"),
+                        "image_id": get_value(ck, "image_id", "img_id"),
+                        "positions": get_value(ck, "positions", "position_int"),
+                        "doc_type": get_value(ck, "doc_type_kwd", "doc_type_kwd"),
+                    }
+                    for ck in ref.get("chunks", [])
+                ]
 
         if not conv.reference:
             conv.reference = []
         conv.reference.append({"chunks": [], "doc_aggs": []})
+
         def stream():
             nonlocal dia, msg, req, conv
             try:
@@ -233,9 +235,7 @@ def completion():
                 ConversationService.update_by_id(conv.id, conv.to_dict())
             except Exception as e:
                 traceback.print_exc()
-                yield "data:" + json.dumps({"code": 500, "message": str(e),
-                                            "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
-                                           ensure_ascii=False) + "\n\n"
+                yield "data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}}, ensure_ascii=False) + "\n\n"
             yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
 
         if req.get("stream", True):
@@ -249,7 +249,7 @@ def completion():
         else:
             answer = None
             for ans in chat(dia, msg, **req):
-                answer = structure_answer(conv, ans, message_id, req["conversation_id"])
+                answer = structure_answer(conv, ans, message_id, conv.id)
                 ConversationService.update_by_id(conv.id, conv.to_dict())
                 break
             return get_json_result(data=answer)
@@ -257,7 +257,7 @@ def completion():
         return server_error_response(e)
 
 
-@manager.route('/tts', methods=['POST'])  # noqa: F821
+@manager.route("/tts", methods=["POST"])  # noqa: F821
 @login_required
 def tts():
     req = request.json
@@ -279,9 +279,7 @@ def tts():
                 for chunk in tts_mdl.tts(txt):
                     yield chunk
         except Exception as e:
-            yield ("data:" + json.dumps({"code": 500, "message": str(e),
-                                         "data": {"answer": "**ERROR**: " + str(e)}},
-                                        ensure_ascii=False)).encode('utf-8')
+            yield ("data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e)}}, ensure_ascii=False)).encode("utf-8")
 
     resp = Response(stream_audio(), mimetype="audio/mpeg")
     resp.headers.add_header("Cache-Control", "no-cache")
@@ -291,7 +289,7 @@ def tts():
     return resp
 
 
-@manager.route('/delete_msg', methods=['POST'])  # noqa: F821
+@manager.route("/delete_msg", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("conversation_id", "message_id")
 def delete_msg():
@@ -314,7 +312,7 @@ def delete_msg():
     return get_json_result(data=conv)
 
 
-@manager.route('/thumbup', methods=['POST'])  # noqa: F821
+@manager.route("/thumbup", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("conversation_id", "message_id")
 def thumbup():
@@ -322,7 +320,7 @@ def thumbup():
     e, conv = ConversationService.get_by_id(req["conversation_id"])
     if not e:
         return get_data_error_result(message="Conversation not found!")
-    up_down = req.get("set")
+    up_down = req.get("thumbup")
     feedback = req.get("feedback", "")
     conv = conv.to_dict()
     for i, msg in enumerate(conv["message"]):
@@ -341,7 +339,7 @@ def thumbup():
     return get_json_result(data=conv)
 
 
-@manager.route('/ask', methods=['POST'])  # noqa: F821
+@manager.route("/ask", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("question", "kb_ids")
 def ask_about():
@@ -354,9 +352,7 @@ def ask_about():
             for ans in ask(req["question"], req["kb_ids"], uid):
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
         except Exception as e:
-            yield "data:" + json.dumps({"code": 500, "message": str(e),
-                                        "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
-                                       ensure_ascii=False) + "\n\n"
+            yield "data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}}, ensure_ascii=False) + "\n\n"
         yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
 
     resp = Response(stream(), mimetype="text/event-stream")
@@ -367,7 +363,7 @@ def ask_about():
     return resp
 
 
-@manager.route('/mindmap', methods=['POST'])  # noqa: F821
+@manager.route("/mindmap", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("question", "kb_ids")
 def mindmap():
@@ -380,18 +376,16 @@ def mindmap():
     embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING, llm_name=kb.embd_id)
     chat_mdl = LLMBundle(current_user.id, LLMType.CHAT)
     question = req["question"]
-    ranks = settings.retrievaler.retrieval(question, embd_mdl, kb.tenant_id, kb_ids, 1, 12,
-                                           0.3, 0.3, aggs=False,
-                                           rank_feature=label_question(question, [kb])
-                                           )
+    ranks = settings.retrievaler.retrieval(question, embd_mdl, kb.tenant_id, kb_ids, 1, 12, 0.3, 0.3, aggs=False, rank_feature=label_question(question, [kb]))
     mindmap = MindMapExtractor(chat_mdl)
-    mind_map = mindmap([c["content_with_weight"] for c in ranks["chunks"]]).output
+    mind_map = trio.run(mindmap, [c["content_with_weight"] for c in ranks["chunks"]])
+    mind_map = mind_map.output
     if "error" in mind_map:
         return server_error_response(Exception(mind_map["error"]))
     return get_json_result(data=mind_map)
 
 
-@manager.route('/related_questions', methods=['POST'])  # noqa: F821
+@manager.route("/related_questions", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("question")
 def related_questions():
@@ -399,31 +393,49 @@ def related_questions():
     question = req["question"]
     chat_mdl = LLMBundle(current_user.id, LLMType.CHAT)
     prompt = """
-Objective: To generate search terms related to the user's search keywords, helping users find more valuable information.
-Instructions:
- - Based on the keywords provided by the user, generate 5-10 related search terms.
- - Each search term should be directly or indirectly related to the keyword, guiding the user to find more valuable information.
- - Use common, general terms as much as possible, avoiding obscure words or technical jargon.
- - Keep the term length between 2-4 words, concise and clear.
- - DO NOT translate, use the language of the original keywords.
+Role: You are an AI language model assistant tasked with generating 5-10 related questions based on a user’s original query. These questions should help expand the search query scope and improve search relevance.
 
-### Example:
-Keywords: Chinese football
-Related search terms:
-1. Current status of Chinese football
-2. Reform of Chinese football
-3. Youth training of Chinese football
-4. Chinese football in the Asian Cup
-5. Chinese football in the World Cup
+Instructions:
+	Input: You are provided with a user’s question.
+	Output: Generate 5-10 alternative questions that are related to the original user question. These alternatives should help retrieve a broader range of relevant documents from a vector database.
+	Context: Focus on rephrasing the original question in different ways, making sure the alternative questions are diverse but still connected to the topic of the original query. Do not create overly obscure, irrelevant, or unrelated questions.
+	Fallback: If you cannot generate any relevant alternatives, do not return any questions.
+	Guidance:
+	1. Each alternative should be unique but still relevant to the original query.
+	2. Keep the phrasing clear, concise, and easy to understand.
+	3. Avoid overly technical jargon or specialized terms unless directly relevant.
+	4. Ensure that each question contributes towards improving search results by broadening the search angle, not narrowing it.
+
+Example:
+Original Question: What are the benefits of electric vehicles?
+
+Alternative Questions:
+	1. How do electric vehicles impact the environment?
+	2. What are the advantages of owning an electric car?
+	3. What is the cost-effectiveness of electric vehicles?
+	4. How do electric vehicles compare to traditional cars in terms of fuel efficiency?
+	5. What are the environmental benefits of switching to electric cars?
+	6. How do electric vehicles help reduce carbon emissions?
+	7. Why are electric vehicles becoming more popular?
+	8. What are the long-term savings of using electric vehicles?
+	9. How do electric vehicles contribute to sustainability?
+	10. What are the key benefits of electric vehicles for consumers?
 
 Reason:
- - When searching, users often only use one or two keywords, making it difficult to fully express their information needs.
- - Generating related search terms can help users dig deeper into relevant information and improve search efficiency. 
- - At the same time, related terms can also help search engines better understand user needs and return more accurate search results.
- 
+	Rephrasing the original query into multiple alternative questions helps the user explore different aspects of their search topic, improving the quality of search results.
+	These questions guide the search engine to provide a more comprehensive set of relevant documents.
 """
-    ans = chat_mdl.chat(prompt, [{"role": "user", "content": f"""
+    ans = chat_mdl.chat(
+        prompt,
+        [
+            {
+                "role": "user",
+                "content": f"""
 Keywords: {question}
 Related search terms:
-    """}], {"temperature": 0.9})
+    """,
+            }
+        ],
+        {"temperature": 0.9},
+    )
     return get_json_result(data=[re.sub(r"^[0-9]\. ", "", a) for a in ans.split("\n") if re.match(r"^[0-9]\. ", a)])

api/apps/dialog_app.py

@@ -18,6 +18,7 @@ from flask import request
 from flask_login import login_required, current_user
 from api.db.services.dialog_service import DialogService
 from api.db import StatusEnum
+from api.db.services.llm_service import TenantLLMService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import TenantService, UserTenantService
 from api import settings
@@ -27,6 +28,7 @@ from api.utils.api_utils import get_json_result
 
 
 @manager.route('/set', methods=['POST'])  # noqa: F821
+@validate_request("prompt_config")
 @login_required
 def set_dialog():
     req = request.json
@@ -42,26 +44,10 @@ def set_dialog():
     similarity_threshold = req.get("similarity_threshold", 0.1)
     vector_similarity_weight = req.get("vector_similarity_weight", 0.3)
     llm_setting = req.get("llm_setting", {})
-    default_prompt = {
-        "system": """你是一个智能助手，请总结知识库的内容来回答问题，请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时，你的回答必须包括“知识库中未找到您要的答案！”这句话。回答需要考虑聊天历史。
-以下是知识库：
-{knowledge}
-以上是知识库。""",
-        "prologue": "您好，我是您的助手小樱，长得可爱又善良，can I help you?",
-        "parameters": [
-            {"key": "knowledge", "optional": False}
-        ],
-        "empty_response": "Sorry! 知识库中未找到相关内容！"
-    }
-    prompt_config = req.get("prompt_config", default_prompt)
-
-    if not prompt_config["system"]:
-        prompt_config["system"] = default_prompt["system"]
-    # if len(prompt_config["parameters"]) < 1:
-    #     prompt_config["parameters"] = default_prompt["parameters"]
-    # for p in prompt_config["parameters"]:
-    #     if p["key"] == "knowledge":break
-    # else: prompt_config["parameters"].append(default_prompt["parameters"][0])
+    prompt_config = req["prompt_config"]
+    
+    if not req.get("kb_ids", []) and not prompt_config.get("tavily_api_key") and "{knowledge}" in prompt_config['system']:
+        return get_data_error_result(message="Please remove `{knowledge}` in system prompt since no knowledge base/Tavily used here.")
 
     for p in prompt_config["parameters"]:
         if p["optional"]:
@@ -74,22 +60,19 @@ def set_dialog():
         e, tenant = TenantService.get_by_id(current_user.id)
         if not e:
             return get_data_error_result(message="Tenant not found!")
-        kbs = KnowledgebaseService.get_by_ids(req.get("kb_ids"))
-        embd_count = len(set([kb.embd_id for kb in kbs]))
-        if embd_count != 1:
+        kbs = KnowledgebaseService.get_by_ids(req.get("kb_ids", []))
+        embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]  # remove vendor suffix for comparison
+        embd_count = len(set(embd_ids))
+        if embd_count > 1:
             return get_data_error_result(message=f'Datasets use different embedding models: {[kb.embd_id for kb in kbs]}"')
 
         llm_id = req.get("llm_id", tenant.llm_id)
         if not dialog_id:
-            if not req.get("kb_ids"):
-                return get_data_error_result(
-                    message="Fail! Please select knowledgebase!")
-
             dia = {
                 "id": get_uuid(),
                 "tenant_id": current_user.id,
                 "name": name,
-                "kb_ids": req["kb_ids"],
+                "kb_ids": req.get("kb_ids", []),
                 "description": description,
                 "llm_id": llm_id,
                 "llm_setting": llm_setting,

api/apps/document_app.py

@@ -20,77 +20,76 @@ import re
 
 import flask
 from flask import request
-from flask_login import login_required, current_user
+from flask_login import current_user, login_required
 
-from deepdoc.parser.html_parser import RAGFlowHtmlParser
-from rag.nlp import search
-
-from api.db import FileType, TaskStatus, ParserType, FileSource
+from api import settings
+from api.constants import FILE_NAME_LEN_LIMIT, IMG_BASE64_PREFIX
+from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileSource, FileType, ParserType, TaskStatus
 from api.db.db_models import File, Task
+from api.db.services import duplicate_name
+from api.db.services.document_service import DocumentService, doc_upload_and_parse
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
-from api.db.services.task_service import queue_tasks
-from api.db.services.user_service import UserTenantService
-from api.db.services import duplicate_name
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.task_service import TaskService
-from api.db.services.document_service import DocumentService, doc_upload_and_parse
+from api.db.services.task_service import TaskService, queue_tasks
+from api.db.services.user_service import UserTenantService
+from api.utils import get_uuid
 from api.utils.api_utils import (
-    server_error_response,
     get_data_error_result,
+    get_json_result,
+    server_error_response,
     validate_request,
 )
-from api.utils import get_uuid
-from api import settings
-from api.utils.api_utils import get_json_result
-from rag.utils.storage_factory import STORAGE_IMPL
-from api.utils.file_utils import filename_type, thumbnail, get_project_base_directory
+from api.utils.file_utils import filename_type, get_project_base_directory, thumbnail
 from api.utils.web_utils import html2pdf, is_valid_url
-from api.constants import IMG_BASE64_PREFIX
+from deepdoc.parser.html_parser import RAGFlowHtmlParser
+from rag.nlp import search
+from rag.utils.storage_factory import STORAGE_IMPL
 
 
-@manager.route('/upload', methods=['POST'])  # noqa: F821
+@manager.route("/upload", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("kb_id")
 def upload():
     kb_id = request.form.get("kb_id")
     if not kb_id:
-        return get_json_result(
-            data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
-    if 'file' not in request.files:
-        return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+    if "file" not in request.files:
+        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
 
-    file_objs = request.files.getlist('file')
+    file_objs = request.files.getlist("file")
     for file_obj in file_objs:
-        if file_obj.filename == '':
-            return get_json_result(
-                data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
+        if file_obj.filename == "":
+            return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
+        if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
 
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
+    err, files = FileService.upload_document(kb, file_objs, current_user.id)
 
-    err, _ = FileService.upload_document(kb, file_objs, current_user.id)
     if err:
-        return get_json_result(
-            data=False, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
-    return get_json_result(data=True)
+        return get_json_result(data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
+
+    if not files:
+        return get_json_result(data=files, message="There seems to be an issue with your file format. Please verify it is correct and not corrupted.", code=settings.RetCode.DATA_ERROR)
+    files = [f[0] for f in files]  # remove the blob
+
+    return get_json_result(data=files)
 
 
-@manager.route('/web_crawl', methods=['POST'])  # noqa: F821
+@manager.route("/web_crawl", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("kb_id", "name", "url")
 def web_crawl():
     kb_id = request.form.get("kb_id")
     if not kb_id:
-        return get_json_result(
-            data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
     name = request.form.get("name")
     url = request.form.get("url")
     if not is_valid_url(url):
-        return get_json_result(
-            data=False, message='The URL format is invalid', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR)
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
@@ -106,10 +105,7 @@ def web_crawl():
     kb_folder = FileService.new_a_file_from_kb(kb.tenant_id, kb.name, kb_root_folder["id"])
 
     try:
-        filename = duplicate_name(
-            DocumentService.query,
-            name=name + ".pdf",
-            kb_id=kb.id)
+        filename = duplicate_name(DocumentService.query, name=name + ".pdf", kb_id=kb.id)
         filetype = filename_type(filename)
         if filetype == FileType.OTHER.value:
             raise RuntimeError("This type of file has not been supported yet!")
@@ -128,7 +124,7 @@ def web_crawl():
             "name": filename,
             "location": location,
             "size": len(blob),
-            "thumbnail": thumbnail(filename, blob)
+            "thumbnail": thumbnail(filename, blob),
         }
         if doc["type"] == FileType.VISUAL:
             doc["parser_id"] = ParserType.PICTURE.value
@@ -145,129 +141,136 @@ def web_crawl():
     return get_json_result(data=True)
 
 
-@manager.route('/create', methods=['POST'])  # noqa: F821
+@manager.route("/create", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("name", "kb_id")
 def create():
     req = request.json
     kb_id = req["kb_id"]
     if not kb_id:
-        return get_json_result(
-            data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+    if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+        return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+
+    if req["name"].strip() == "":
+        return get_json_result(data=False, message="File name can't be empty.", code=settings.RetCode.ARGUMENT_ERROR)
+    req["name"] = req["name"].strip()
 
     try:
         e, kb = KnowledgebaseService.get_by_id(kb_id)
         if not e:
-            return get_data_error_result(
-                message="Can't find this knowledgebase!")
+            return get_data_error_result(message="Can't find this knowledgebase!")
 
         if DocumentService.query(name=req["name"], kb_id=kb_id):
-            return get_data_error_result(
-                message="Duplicated document name in the same knowledgebase.")
+            return get_data_error_result(message="Duplicated document name in the same knowledgebase.")
 
-        doc = DocumentService.insert({
-            "id": get_uuid(),
-            "kb_id": kb.id,
-            "parser_id": kb.parser_id,
-            "parser_config": kb.parser_config,
-            "created_by": current_user.id,
-            "type": FileType.VIRTUAL,
-            "name": req["name"],
-            "location": "",
-            "size": 0
-        })
+        doc = DocumentService.insert(
+            {
+                "id": get_uuid(),
+                "kb_id": kb.id,
+                "parser_id": kb.parser_id,
+                "parser_config": kb.parser_config,
+                "created_by": current_user.id,
+                "type": FileType.VIRTUAL,
+                "name": req["name"],
+                "location": "",
+                "size": 0,
+            }
+        )
         return get_json_result(data=doc.to_json())
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/list', methods=['GET'])  # noqa: F821
+@manager.route("/list", methods=["POST"])  # noqa: F821
 @login_required
 def list_docs():
     kb_id = request.args.get("kb_id")
     if not kb_id:
-        return get_json_result(
-            data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
     tenants = UserTenantService.query(user_id=current_user.id)
     for tenant in tenants:
-        if KnowledgebaseService.query(
-                tenant_id=tenant.tenant_id, id=kb_id):
+        if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
             break
     else:
-        return get_json_result(
-            data=False, message='Only owner of knowledgebase authorized for this operation.',
-            code=settings.RetCode.OPERATING_ERROR)
+        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
     keywords = request.args.get("keywords", "")
 
-    page_number = int(request.args.get("page", 1))
-    items_per_page = int(request.args.get("page_size", 15))
+    page_number = int(request.args.get("page", 0))
+    items_per_page = int(request.args.get("page_size", 0))
     orderby = request.args.get("orderby", "create_time")
-    desc = request.args.get("desc", True)
+    if request.args.get("desc", "true").lower() == "false":
+        desc = False
+    else:
+        desc = True
+
+    req = request.get_json()
+
+    run_status = req.get("run_status", [])
+    if run_status:
+        invalid_status = {s for s in run_status if s not in VALID_TASK_STATUS}
+        if invalid_status:
+            return get_data_error_result(message=f"Invalid filter run status conditions: {', '.join(invalid_status)}")
+
+    types = req.get("types", [])
+    if types:
+        invalid_types = {t for t in types if t not in VALID_FILE_TYPES}
+        if invalid_types:
+            return get_data_error_result(message=f"Invalid filter conditions: {', '.join(invalid_types)} type{'s' if len(invalid_types) > 1 else ''}")
+
     try:
-        docs, tol = DocumentService.get_by_kb_id(
-            kb_id, page_number, items_per_page, orderby, desc, keywords)
+        docs, tol = DocumentService.get_by_kb_id(kb_id, page_number, items_per_page, orderby, desc, keywords, run_status, types)
 
         for doc_item in docs:
-            if doc_item['thumbnail'] and not doc_item['thumbnail'].startswith(IMG_BASE64_PREFIX):
-                doc_item['thumbnail'] = f"/v1/document/image/{kb_id}-{doc_item['thumbnail']}"
+            if doc_item["thumbnail"] and not doc_item["thumbnail"].startswith(IMG_BASE64_PREFIX):
+                doc_item["thumbnail"] = f"/v1/document/image/{kb_id}-{doc_item['thumbnail']}"
 
         return get_json_result(data={"total": tol, "docs": docs})
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/infos', methods=['POST'])  # noqa: F821
+@manager.route("/infos", methods=["POST"])  # noqa: F821
 @login_required
 def docinfos():
     req = request.json
     doc_ids = req["doc_ids"]
     for doc_id in doc_ids:
         if not DocumentService.accessible(doc_id, current_user.id):
-            return get_json_result(
-                data=False,
-                message='No authorization.',
-                code=settings.RetCode.AUTHENTICATION_ERROR
-            )
+            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
     docs = DocumentService.get_by_ids(doc_ids)
     return get_json_result(data=list(docs.dicts()))
 
 
-@manager.route('/thumbnails', methods=['GET'])  # noqa: F821
+@manager.route("/thumbnails", methods=["GET"])  # noqa: F821
 # @login_required
 def thumbnails():
     doc_ids = request.args.get("doc_ids").split(",")
     if not doc_ids:
-        return get_json_result(
-            data=False, message='Lack of "Document ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "Document ID"', code=settings.RetCode.ARGUMENT_ERROR)
 
     try:
         docs = DocumentService.get_thumbnails(doc_ids)
 
         for doc_item in docs:
-            if doc_item['thumbnail'] and not doc_item['thumbnail'].startswith(IMG_BASE64_PREFIX):
-                doc_item['thumbnail'] = f"/v1/document/image/{doc_item['kb_id']}-{doc_item['thumbnail']}"
+            if doc_item["thumbnail"] and not doc_item["thumbnail"].startswith(IMG_BASE64_PREFIX):
+                doc_item["thumbnail"] = f"/v1/document/image/{doc_item['kb_id']}-{doc_item['thumbnail']}"
 
         return get_json_result(data={d["id"]: d["thumbnail"] for d in docs})
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/change_status', methods=['POST'])  # noqa: F821
+@manager.route("/change_status", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_id", "status")
 def change_status():
     req = request.json
     if str(req["status"]) not in ["0", "1"]:
-        return get_json_result(
-            data=False,
-            message='"Status" must be either 0 or 1!',
-            code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='"Status" must be either 0 or 1!', code=settings.RetCode.ARGUMENT_ERROR)
 
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(
-            data=False,
-            message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
 
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
@@ -275,23 +278,19 @@ def change_status():
             return get_data_error_result(message="Document not found!")
         e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
         if not e:
-            return get_data_error_result(
-                message="Can't find this knowledgebase!")
+            return get_data_error_result(message="Can't find this knowledgebase!")
 
-        if not DocumentService.update_by_id(
-                req["doc_id"], {"status": str(req["status"])}):
-            return get_data_error_result(
-                message="Database error (Document update)!")
+        if not DocumentService.update_by_id(req["doc_id"], {"status": str(req["status"])}):
+            return get_data_error_result(message="Database error (Document update)!")
 
         status = int(req["status"])
-        settings.docStoreConn.update({"doc_id": req["doc_id"]}, {"available_int": status},
-                                     search.index_name(kb.tenant_id), doc.kb_id)
+        settings.docStoreConn.update({"doc_id": req["doc_id"]}, {"available_int": status}, search.index_name(kb.tenant_id), doc.kb_id)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/rm', methods=['POST'])  # noqa: F821
+@manager.route("/rm", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_id")
 def rm():
@@ -302,16 +301,13 @@ def rm():
 
     for doc_id in doc_ids:
         if not DocumentService.accessible4deletion(doc_id, current_user.id):
-            return get_json_result(
-                data=False,
-                message='No authorization.',
-                code=settings.RetCode.AUTHENTICATION_ERROR
-            )
+            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
 
     root_folder = FileService.get_root_folder(current_user.id)
     pf_id = root_folder["id"]
     FileService.init_knowledgebase_docs(pf_id, current_user.id)
     errors = ""
+    kb_table_num_map = {}
     for doc_id in doc_ids:
         try:
             e, doc = DocumentService.get_by_id(doc_id)
@@ -325,14 +321,25 @@ def rm():
 
             TaskService.filter_delete([Task.doc_id == doc_id])
             if not DocumentService.remove_document(doc, tenant_id):
-                return get_data_error_result(
-                    message="Database error (Document removal)!")
+                return get_data_error_result(message="Database error (Document removal)!")
 
             f2d = File2DocumentService.get_by_document_id(doc_id)
-            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
+            deleted_file_count = 0
+            if f2d:
+                deleted_file_count = FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
             File2DocumentService.delete_by_document_id(doc_id)
+            if deleted_file_count > 0:
+                STORAGE_IMPL.rm(b, n)
 
-            STORAGE_IMPL.rm(b, n)
+            doc_parser = doc.parser_id
+            if doc_parser == ParserType.TABLE:
+                kb_id = doc.kb_id
+                if kb_id not in kb_table_num_map:
+                    counts = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
+                    kb_table_num_map[kb_id] = counts
+                kb_table_num_map[kb_id] -= 1
+                if kb_table_num_map[kb_id] <= 0:
+                    KnowledgebaseService.delete_field_map(kb_id)
         except Exception as e:
             errors += str(e)
 
@@ -342,19 +349,16 @@ def rm():
     return get_json_result(data=True)
 
 
-@manager.route('/run', methods=['POST'])  # noqa: F821
+@manager.route("/run", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_ids", "run")
 def run():
     req = request.json
     for doc_id in req["doc_ids"]:
         if not DocumentService.accessible(doc_id, current_user.id):
-            return get_json_result(
-                data=False,
-                message='No authorization.',
-                code=settings.RetCode.AUTHENTICATION_ERROR
-            )
+            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
     try:
+        kb_table_num_map = {}
         for id in req["doc_ids"]:
             info = {"run": str(req["run"]), "progress": 0}
             if str(req["run"]) == TaskStatus.RUNNING.value and req.get("delete", False):
@@ -377,44 +381,47 @@ def run():
                 e, doc = DocumentService.get_by_id(id)
                 doc = doc.to_dict()
                 doc["tenant_id"] = tenant_id
+
+                doc_parser = doc.get("parser_id", ParserType.NAIVE)
+                if doc_parser == ParserType.TABLE:
+                    kb_id = doc.get("kb_id")
+                    if not kb_id:
+                        continue
+                    if kb_id not in kb_table_num_map:
+                        count = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
+                        kb_table_num_map[kb_id] = count
+                        if kb_table_num_map[kb_id] <= 0:
+                            KnowledgebaseService.delete_field_map(kb_id)
                 bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
-                queue_tasks(doc, bucket, name)
+                queue_tasks(doc, bucket, name, 0)
 
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/rename', methods=['POST'])  # noqa: F821
+@manager.route("/rename", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_id", "name")
 def rename():
     req = request.json
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(
-            data=False,
-            message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
-        )
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
             return get_data_error_result(message="Document not found!")
-        if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
-                doc.name.lower()).suffix:
-            return get_json_result(
-                data=False,
-                message="The extension of file can't be changed",
-                code=settings.RetCode.ARGUMENT_ERROR)
+        if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
+            return get_json_result(data=False, message="The extension of file can't be changed", code=settings.RetCode.ARGUMENT_ERROR)
+        if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+
         for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
             if d.name == req["name"]:
-                return get_data_error_result(
-                    message="Duplicated document name in the same knowledgebase.")
+                return get_data_error_result(message="Duplicated document name in the same knowledgebase.")
 
-        if not DocumentService.update_by_id(
-                req["doc_id"], {"name": req["name"]}):
-            return get_data_error_result(
-                message="Database error (Document rename)!")
+        if not DocumentService.update_by_id(req["doc_id"], {"name": req["name"]}):
+            return get_data_error_result(message="Database error (Document rename)!")
 
         informs = File2DocumentService.get_by_document_id(req["doc_id"])
         if informs:
@@ -426,7 +433,7 @@ def rename():
         return server_error_response(e)
 
 
-@manager.route('/get/<doc_id>', methods=['GET'])  # noqa: F821
+@manager.route("/get/<doc_id>", methods=["GET"])  # noqa: F821
 # @login_required
 def get(doc_id):
     try:
@@ -440,29 +447,22 @@ def get(doc_id):
         ext = re.search(r"\.([^.]+)$", doc.name)
         if ext:
             if doc.type == FileType.VISUAL.value:
-                response.headers.set('Content-Type', 'image/%s' % ext.group(1))
+                response.headers.set("Content-Type", "image/%s" % ext.group(1))
             else:
-                response.headers.set(
-                    'Content-Type',
-                    'application/%s' %
-                    ext.group(1))
+                response.headers.set("Content-Type", "application/%s" % ext.group(1))
         return response
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/change_parser', methods=['POST'])  # noqa: F821
+@manager.route("/change_parser", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_id", "parser_id")
 def change_parser():
     req = request.json
 
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(
-            data=False,
-            message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
-        )
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
@@ -474,21 +474,16 @@ def change_parser():
             else:
                 return get_json_result(data=True)
 
-        if ((doc.type == FileType.VISUAL and req["parser_id"] != "picture")
-                or (re.search(
-                    r"\.(ppt|pptx|pages)$", doc.name) and req["parser_id"] != "presentation")):
+        if (doc.type == FileType.VISUAL and req["parser_id"] != "picture") or (re.search(r"\.(ppt|pptx|pages)$", doc.name) and req["parser_id"] != "presentation"):
             return get_data_error_result(message="Not supported yet!")
 
-        e = DocumentService.update_by_id(doc.id,
-                                         {"parser_id": req["parser_id"], "progress": 0, "progress_msg": "",
-                                          "run": TaskStatus.UNSTART.value})
+        e = DocumentService.update_by_id(doc.id, {"parser_id": req["parser_id"], "progress": 0, "progress_msg": "", "run": TaskStatus.UNSTART.value})
         if not e:
             return get_data_error_result(message="Document not found!")
         if "parser_config" in req:
             DocumentService.update_parser_config(doc.id, req["parser_config"])
         if doc.token_num > 0:
-            e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1,
-                                                    doc.process_duation * -1)
+            e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, doc.process_duation * -1)
             if not e:
                 return get_data_error_result(message="Document not found!")
             tenant_id = DocumentService.get_tenant_id(req["doc_id"])
@@ -502,7 +497,7 @@ def change_parser():
         return server_error_response(e)
 
 
-@manager.route('/image/<image_id>', methods=['GET'])  # noqa: F821
+@manager.route("/image/<image_id>", methods=["GET"])  # noqa: F821
 # @login_required
 def get_image(image_id):
     try:
@@ -511,53 +506,46 @@ def get_image(image_id):
             return get_data_error_result(message="Image not found.")
         bkt, nm = image_id.split("-")
         response = flask.make_response(STORAGE_IMPL.get(bkt, nm))
-        response.headers.set('Content-Type', 'image/JPEG')
+        response.headers.set("Content-Type", "image/JPEG")
         return response
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/upload_and_parse', methods=['POST'])  # noqa: F821
+@manager.route("/upload_and_parse", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("conversation_id")
 def upload_and_parse():
-    if 'file' not in request.files:
-        return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+    if "file" not in request.files:
+        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
 
-    file_objs = request.files.getlist('file')
+    file_objs = request.files.getlist("file")
     for file_obj in file_objs:
-        if file_obj.filename == '':
-            return get_json_result(
-                data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
+        if file_obj.filename == "":
+            return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
 
     doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, current_user.id)
 
     return get_json_result(data=doc_ids)
 
 
-@manager.route('/parse', methods=['POST'])  # noqa: F821
+@manager.route("/parse", methods=["POST"])  # noqa: F821
 @login_required
 def parse():
     url = request.json.get("url") if request.json else ""
     if url:
         if not is_valid_url(url):
-            return get_json_result(
-                data=False, message='The URL format is invalid', code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR)
         download_path = os.path.join(get_project_base_directory(), "logs/downloads")
         os.makedirs(download_path, exist_ok=True)
         from seleniumwire.webdriver import Chrome, ChromeOptions
+
         options = ChromeOptions()
-        options.add_argument('--headless')
-        options.add_argument('--disable-gpu')
-        options.add_argument('--no-sandbox')
-        options.add_argument('--disable-dev-shm-usage')
-        options.add_experimental_option('prefs', {
-            'download.default_directory': download_path,
-            'download.prompt_for_download': False,
-            'download.directory_upgrade': True,
-            'safebrowsing.enabled': True
-        })
+        options.add_argument("--headless")
+        options.add_argument("--disable-gpu")
+        options.add_argument("--no-sandbox")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_experimental_option("prefs", {"download.default_directory": download_path, "download.prompt_for_download": False, "download.directory_upgrade": True, "safebrowsing.enabled": True})
         driver = Chrome(options=options)
         driver.get(url)
         res_headers = [r.response.headers for r in driver.requests if r and r.response]
@@ -580,51 +568,41 @@ def parse():
 
         r = re.search(r"filename=\"([^\"]+)\"", str(res_headers))
         if not r or not r.group(1):
-            return get_json_result(
-                data=False, message="Can't not identify downloaded file", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="Can't not identify downloaded file", code=settings.RetCode.ARGUMENT_ERROR)
         f = File(r.group(1), os.path.join(download_path, r.group(1)))
         txt = FileService.parse_docs([f], current_user.id)
         return get_json_result(data=txt)
 
-    if 'file' not in request.files:
-        return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+    if "file" not in request.files:
+        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
 
-    file_objs = request.files.getlist('file')
+    file_objs = request.files.getlist("file")
     txt = FileService.parse_docs(file_objs, current_user.id)
 
     return get_json_result(data=txt)
 
 
-@manager.route('/set_meta', methods=['POST'])  # noqa: F821
+@manager.route("/set_meta", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_id", "meta")
 def set_meta():
     req = request.json
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(
-            data=False,
-            message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
-        )
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
     try:
         meta = json.loads(req["meta"])
     except Exception as e:
-        return get_json_result(
-            data=False, message=f'Json syntax error: {e}', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message=f"Json syntax error: {e}", code=settings.RetCode.ARGUMENT_ERROR)
     if not isinstance(meta, dict):
-        return get_json_result(
-            data=False, message='Meta data should be in Json map format, like {"key": "value"}', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Meta data should be in Json map format, like {"key": "value"}', code=settings.RetCode.ARGUMENT_ERROR)
 
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
             return get_data_error_result(message="Document not found!")
 
-        if not DocumentService.update_by_id(
-                req["doc_id"], {"meta_fields": meta}):
-            return get_data_error_result(
-                message="Database error (meta updates)!")
+        if not DocumentService.update_by_id(req["doc_id"], {"meta_fields": meta}):
+            return get_data_error_result(message="Database error (meta updates)!")
 
         return get_json_result(data=True)
     except Exception as e:

api/apps/file2document_app.py

@@ -38,8 +38,12 @@ def convert():
     file2documents = []
 
     try:
+        files = FileService.get_by_ids(file_ids)
+        files_set = dict({file.id: file for file in files})
         for file_id in file_ids:
-            e, file = FileService.get_by_id(file_id)
+            file = files_set[file_id]
+            if not file:
+                return get_data_error_result(message="File not found!")
             file_ids_list = [file_id]
             if file.type == FileType.FOLDER.value:
                 file_ids_list = FileService.get_all_innermost_file_ids(file_id, [])
@@ -86,6 +90,7 @@ def convert():
                         "file_id": id,
                         "document_id": doc.id,
                     })
+
                     file2documents.append(file2document.to_json())
         return get_json_result(data=file2documents)
     except Exception as e:

api/apps/file_app.py

@@ -55,20 +55,17 @@ def upload():
                 data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
     file_res = []
     try:
+        e, pf_folder = FileService.get_by_id(pf_id)
+        if not e:
+            return get_data_error_result( message="Can't find this folder!")
         for file_obj in file_objs:
-            e, file = FileService.get_by_id(pf_id)
-            if not e:
-                return get_data_error_result(
-                    message="Can't find this folder!")
             MAX_FILE_NUM_PER_USER = int(os.environ.get('MAX_FILE_NUM_PER_USER', 0))
             if MAX_FILE_NUM_PER_USER > 0 and DocumentService.get_doc_count(current_user.id) >= MAX_FILE_NUM_PER_USER:
-                return get_data_error_result(
-                    message="Exceed the maximum file number of a free user!")
+                return get_data_error_result( message="Exceed the maximum file number of a free user!")
 
             # split file name path
             if not file_obj.filename:
-                e, file = FileService.get_by_id(pf_id)
-                file_obj_names = [file.name, file_obj.filename]
+                file_obj_names = [pf_folder.name, file_obj.filename]
             else:
                 full_path = '/' + file_obj.filename
                 file_obj_names = full_path.split('/')
@@ -184,7 +181,7 @@ def list_files():
             current_user.id, pf_id, page_number, items_per_page, orderby, desc, keywords)
 
         parent_folder = FileService.get_parent_folder(pf_id)
-        if not FileService.get_parent_folder(pf_id):
+        if not parent_folder:
             return get_json_result(message="File not found!")
 
         return get_json_result(data={"total": total, "files": files, "parent_folder": parent_folder.to_json()})
@@ -260,6 +257,7 @@ def rm():
                     STORAGE_IMPL.rm(file.parent_id, file.location)
                 FileService.delete_folder_by_pf_id(current_user.id, file_id)
             else:
+                STORAGE_IMPL.rm(file.parent_id, file.location)
                 if not FileService.delete(file):
                     return get_data_error_result(
                         message="Database error (File removal)!")
@@ -358,9 +356,14 @@ def move():
     try:
         file_ids = req["src_file_ids"]
         parent_id = req["dest_file_id"]
+        files = FileService.get_by_ids(file_ids)
+        files_dict = {}
+        for file in files:
+            files_dict[file.id] = file
+
         for file_id in file_ids:
-            e, file = FileService.get_by_id(file_id)
-            if not e:
+            file = files_dict[file_id]
+            if not file:
                 return get_data_error_result(message="File or Folder not found!")
             if not file.tenant_id:
                 return get_data_error_result(message="Tenant not found!")

api/apps/kb_app.py

@@ -14,7 +14,6 @@
 #  limitations under the License.
 #
 import json
-import logging
 import os
 
 from flask import request
@@ -35,6 +34,7 @@ from api import settings
 from rag.nlp import search
 from api.constants import DATASET_NAME_LIMIT
 from rag.settings import PAGERANK_FLD
+from rag.utils.storage_factory import STORAGE_IMPL
 
 
 @manager.route('/create', methods=['post'])  # noqa: F821
@@ -45,11 +45,11 @@ def create():
     dataset_name = req["name"]
     if not isinstance(dataset_name, str):
         return get_data_error_result(message="Dataset name must be string.")
-    if dataset_name == "":
+    if dataset_name.strip() == "":
         return get_data_error_result(message="Dataset name can't be empty.")
-    if len(dataset_name) >= DATASET_NAME_LIMIT:
+    if len(dataset_name.encode("utf-8")) > DATASET_NAME_LIMIT:
         return get_data_error_result(
-            message=f"Dataset name length is {len(dataset_name)} which is large than {DATASET_NAME_LIMIT}")
+            message=f"Dataset name length is {len(dataset_name)} which is larger than {DATASET_NAME_LIMIT}")
 
     dataset_name = dataset_name.strip()
     dataset_name = duplicate_name(
@@ -59,6 +59,7 @@ def create():
         status=StatusEnum.VALID.value)
     try:
         req["id"] = get_uuid()
+        req["name"] = dataset_name
         req["tenant_id"] = current_user.id
         req["created_by"] = current_user.id
         e, t = TenantService.get_by_id(current_user.id)
@@ -74,11 +75,19 @@ def create():
 
 @manager.route('/update', methods=['post'])  # noqa: F821
 @login_required
-@validate_request("kb_id", "name", "description", "permission", "parser_id")
+@validate_request("kb_id", "name", "description", "parser_id")
 @not_allowed_parameters("id", "tenant_id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by")
 def update():
     req = request.json
+    if not isinstance(req["name"], str):
+        return get_data_error_result(message="Dataset name must be string.")
+    if req["name"].strip() == "":
+        return get_data_error_result(message="Dataset name can't be empty.")
+    if len(req["name"].encode("utf-8")) > DATASET_NAME_LIMIT:
+        return get_data_error_result(
+            message=f"Dataset name length is {len(req['name'])} which is large than {DATASET_NAME_LIMIT}")
     req["name"] = req["name"].strip()
+
     if not KnowledgebaseService.accessible4deletion(req["kb_id"], current_user.id):
         return get_json_result(
             data=False,
@@ -100,13 +109,13 @@ def update():
         if req.get("parser_id", "") == "tag" and os.environ.get('DOC_ENGINE', "elasticsearch") == "infinity":
             return get_json_result(
                 data=False,
-                message='The chunk method Tag has not been supported by Infinity yet.',
+                message='The chunking method Tag has not been supported by Infinity yet.',
                 code=settings.RetCode.OPERATING_ERROR
             )
 
         if req["name"].lower() != kb.name.lower() \
                 and len(
-            KnowledgebaseService.query(name=req["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) > 1:
+            KnowledgebaseService.query(name=req["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) >= 1:
             return get_data_error_result(
                 message="Duplicated knowledgebase name.")
 
@@ -115,6 +124,9 @@ def update():
             return get_data_error_result()
 
         if kb.pagerank != req.get("pagerank", 0):
+            if os.environ.get("DOC_ENGINE", "elasticsearch") != "elasticsearch":
+                return get_data_error_result(message="'pagerank' can only be set when doc_engine is elasticsearch")
+            
             if req.get("pagerank", 0) > 0:
                 settings.docStoreConn.update({"kb_id": kb.id}, {PAGERANK_FLD: req["pagerank"]},
                                          search.index_name(kb.tenant_id), kb.id)
@@ -153,30 +165,47 @@ def detail():
         if not kb:
             return get_data_error_result(
                 message="Can't find this knowledgebase!")
+        kb["size"] = DocumentService.get_total_size_by_kb_id(kb_id=kb["id"],keywords="", run_status=[], types=[])
         return get_json_result(data=kb)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/list', methods=['GET'])  # noqa: F821
+@manager.route('/list', methods=['POST'])  # noqa: F821
 @login_required
 def list_kbs():
     keywords = request.args.get("keywords", "")
-    page_number = int(request.args.get("page", 1))
-    items_per_page = int(request.args.get("page_size", 150))
+    page_number = int(request.args.get("page", 0))
+    items_per_page = int(request.args.get("page_size", 0))
     parser_id = request.args.get("parser_id")
     orderby = request.args.get("orderby", "create_time")
-    desc = request.args.get("desc", True)
+    if request.args.get("desc", "true").lower() == "false":
+        desc = False
+    else:
+        desc = True
+
+    req = request.get_json()
+    owner_ids = req.get("owner_ids", [])
     try:
-        tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
-        kbs, total = KnowledgebaseService.get_by_tenant_ids(
-            [m["tenant_id"] for m in tenants], current_user.id, page_number,
-            items_per_page, orderby, desc, keywords, parser_id)
+        if not owner_ids:
+            tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
+            tenants = [m["tenant_id"] for m in tenants]
+            kbs, total = KnowledgebaseService.get_by_tenant_ids(
+                tenants, current_user.id, page_number,
+                items_per_page, orderby, desc, keywords, parser_id)
+        else:
+            tenants = owner_ids
+            kbs, total = KnowledgebaseService.get_by_tenant_ids(
+                tenants, current_user.id, 0,
+                0, orderby, desc, keywords, parser_id)
+            kbs = [kb for kb in kbs if kb["tenant_id"] in tenants]
+            total = len(kbs)
+            if page_number and items_per_page:
+                kbs = kbs[(page_number-1)*items_per_page:page_number*items_per_page]
         return get_json_result(data={"kbs": kbs, "total": total})
     except Exception as e:
         return server_error_response(e)
 
-
 @manager.route('/rm', methods=['post'])  # noqa: F821
 @login_required
 @validate_request("kb_id")
@@ -212,6 +241,8 @@ def rm():
         for kb in kbs:
             settings.docStoreConn.delete({"kb_id": kb.id}, search.index_name(kb.tenant_id), kb.id)
             settings.docStoreConn.deleteIdx(search.index_name(kb.tenant_id), kb.id)
+            if hasattr(STORAGE_IMPL, 'remove_bucket'):
+                STORAGE_IMPL.remove_bucket(kb.id)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
@@ -300,11 +331,12 @@ def knowledge_graph(kb_id):
         "kb_id": [kb_id],
         "knowledge_graph_kwd": ["graph"]
     }
+
     obj = {"graph": {}, "mind_map": {}}
-    try:
-        sres = settings.retrievaler.search(req, search.index_name(kb.tenant_id), [kb_id])
-    except Exception as e:
-        logging.exception(e)
+    if not settings.docStoreConn.indexExist(search.index_name(kb.tenant_id), kb_id):
+        return get_json_result(data=obj)
+    sres = settings.retrievaler.search(req, search.index_name(kb.tenant_id), [kb_id])
+    if not len(sres.ids):
         return get_json_result(data=obj)
 
     for id in sres.ids[:1]:
@@ -318,6 +350,22 @@ def knowledge_graph(kb_id):
 
     if "nodes" in obj["graph"]:
         obj["graph"]["nodes"] = sorted(obj["graph"]["nodes"], key=lambda x: x.get("pagerank", 0), reverse=True)[:256]
-    if "edges" in obj["graph"]:
-        obj["graph"]["edges"] = sorted(obj["graph"]["edges"], key=lambda x: x.get("weight", 0), reverse=True)[:128]
-    return get_json_result(data=obj)
+        if "edges" in obj["graph"]:
+            node_id_set = { o["id"] for o in obj["graph"]["nodes"] }
+            filtered_edges = [o for o in obj["graph"]["edges"] if o["source"] != o["target"] and o["source"] in node_id_set and o["target"] in node_id_set]
+            obj["graph"]["edges"] = sorted(filtered_edges, key=lambda x: x.get("weight", 0), reverse=True)[:128]
+    return get_json_result(data=obj)
+
+@manager.route('/<kb_id>/knowledge_graph', methods=['DELETE'])  # noqa: F821
+@login_required
+def delete_knowledge_graph(kb_id):
+    if not KnowledgebaseService.accessible(kb_id, current_user.id):
+        return get_json_result(
+            data=False,
+            message='No authorization.',
+            code=settings.RetCode.AUTHENTICATION_ERROR
+        )
+    _, kb = KnowledgebaseService.get_by_id(kb_id)
+    settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), kb_id)
+
+    return get_json_result(data=True)

api/apps/langfuse_app.py

@@ -0,0 +1,97 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+from flask import request
+from flask_login import current_user, login_required
+from langfuse import Langfuse
+
+from api.db.db_models import DB
+from api.db.services.langfuse_service import TenantLangfuseService
+from api.utils.api_utils import get_error_data_result, get_json_result, server_error_response, validate_request
+
+
+@manager.route("/api_key", methods=["POST", "PUT"])  # noqa: F821
+@login_required
+@validate_request("secret_key", "public_key", "host")
+def set_api_key():
+    req = request.get_json()
+    secret_key = req.get("secret_key", "")
+    public_key = req.get("public_key", "")
+    host = req.get("host", "")
+    if not all([secret_key, public_key, host]):
+        return get_error_data_result(message="Missing required fields")
+
+    langfuse_keys = dict(
+        tenant_id=current_user.id,
+        secret_key=secret_key,
+        public_key=public_key,
+        host=host,
+    )
+
+    langfuse = Langfuse(public_key=langfuse_keys["public_key"], secret_key=langfuse_keys["secret_key"], host=langfuse_keys["host"])
+    if not langfuse.auth_check():
+        return get_error_data_result(message="Invalid Langfuse keys")
+
+    langfuse_entry = TenantLangfuseService.filter_by_tenant(tenant_id=current_user.id)
+    with DB.atomic():
+        try:
+            if not langfuse_entry:
+                TenantLangfuseService.save(**langfuse_keys)
+            else:
+                TenantLangfuseService.update_by_tenant(tenant_id=current_user.id, langfuse_keys=langfuse_keys)
+            return get_json_result(data=langfuse_keys)
+        except Exception as e:
+            server_error_response(e)
+
+
+@manager.route("/api_key", methods=["GET"])  # noqa: F821
+@login_required
+@validate_request()
+def get_api_key():
+    langfuse_entry = TenantLangfuseService.filter_by_tenant_with_info(tenant_id=current_user.id)
+    if not langfuse_entry:
+        return get_json_result(message="Have not record any Langfuse keys.")
+
+    langfuse = Langfuse(public_key=langfuse_entry["public_key"], secret_key=langfuse_entry["secret_key"], host=langfuse_entry["host"])
+    try:
+        if not langfuse.auth_check():
+            return get_error_data_result(message="Invalid Langfuse keys loaded")
+    except langfuse.api.core.api_error.ApiError as api_err:
+        return get_json_result(message=f"Error from Langfuse: {api_err}")
+    except Exception as e:
+        server_error_response(e)
+
+    langfuse_entry["project_id"] = langfuse.api.projects.get().dict()["data"][0]["id"]
+    langfuse_entry["project_name"] = langfuse.api.projects.get().dict()["data"][0]["name"]
+
+    return get_json_result(data=langfuse_entry)
+
+
+@manager.route("/api_key", methods=["DELETE"])  # noqa: F821
+@login_required
+@validate_request()
+def delete_api_key():
+    langfuse_entry = TenantLangfuseService.filter_by_tenant(tenant_id=current_user.id)
+    if not langfuse_entry:
+        return get_json_result(message="Have not record any Langfuse keys.")
+
+    with DB.atomic():
+        try:
+            TenantLangfuseService.delete_model(langfuse_entry)
+            return get_json_result(data=True)
+        except Exception as e:
+            server_error_response(e)

api/apps/llm_app.py

@@ -61,6 +61,7 @@ def set_api_key():
     msg = ""
     for llm in LLMService.query(fid=factory):
         if not embd_passed and llm.model_type == LLMType.EMBEDDING.value:
+            assert factory in EmbeddingModel, f"Embedding model from {factory} is not supported yet."
             mdl = EmbeddingModel[factory](
                 req["api_key"], llm.llm_name, base_url=req.get("base_url"))
             try:
@@ -71,6 +72,7 @@ def set_api_key():
             except Exception as e:
                 msg += f"\nFail to access embedding model({llm.llm_name}) using this api key." + str(e)
         elif not chat_passed and llm.model_type == LLMType.CHAT.value:
+            assert factory in ChatModel, f"Chat model from {factory} is not supported yet."
             mdl = ChatModel[factory](
                 req["api_key"], llm.llm_name, base_url=req.get("base_url"))
             try:
@@ -83,6 +85,7 @@ def set_api_key():
                 msg += f"\nFail to access model({llm.llm_name}) using this api key." + str(
                     e)
         elif not rerank_passed and llm.model_type == LLMType.RERANK:
+            assert factory in RerankModel, f"Re-rank model from {factory} is not supported yet."
             mdl = RerankModel[factory](
                 req["api_key"], llm.llm_name, base_url=req.get("base_url"))
             try:
@@ -135,6 +138,8 @@ def set_api_key():
 def add_llm():
     req = request.json
     factory = req["llm_factory"]
+    api_key = req.get("api_key", "x")
+    llm_name = req.get("llm_name")
 
     def apikey_json(keys):
         nonlocal req
@@ -143,7 +148,6 @@ def add_llm():
     if factory == "VolcEngine":
         # For VolcEngine, due to its special authentication method
         # Assemble ark_api_key endpoint_id into api_key
-        llm_name = req["llm_name"]
         api_key = apikey_json(["ark_api_key", "endpoint_id"])
 
     elif factory == "Tencent Hunyuan":
@@ -152,52 +156,43 @@ def add_llm():
 
     elif factory == "Tencent Cloud":
         req["api_key"] = apikey_json(["tencent_cloud_sid", "tencent_cloud_sk"])
+        return set_api_key()
 
     elif factory == "Bedrock":
         # For Bedrock, due to its special authentication method
         # Assemble bedrock_ak, bedrock_sk, bedrock_region
-        llm_name = req["llm_name"]
         api_key = apikey_json(["bedrock_ak", "bedrock_sk", "bedrock_region"])
 
     elif factory == "LocalAI":
-        llm_name = req["llm_name"] + "___LocalAI"
-        api_key = "xxxxxxxxxxxxxxx"
+        llm_name += "___LocalAI"
 
     elif factory == "HuggingFace":
-        llm_name = req["llm_name"] + "___HuggingFace"
-        api_key = "xxxxxxxxxxxxxxx"
+        llm_name += "___HuggingFace"
 
     elif factory == "OpenAI-API-Compatible":
-        llm_name = req["llm_name"] + "___OpenAI-API"
-        api_key = req.get("api_key", "xxxxxxxxxxxxxxx")
+        llm_name += "___OpenAI-API"
+
+    elif factory == "VLLM":
+        llm_name += "___VLLM"
 
     elif factory == "XunFei Spark":
-        llm_name = req["llm_name"]
         if req["model_type"] == "chat":
-            api_key = req.get("spark_api_password", "xxxxxxxxxxxxxxx")
+            api_key = req.get("spark_api_password", "")
         elif req["model_type"] == "tts":
             api_key = apikey_json(["spark_app_id", "spark_api_secret", "spark_api_key"])
 
     elif factory == "BaiduYiyan":
-        llm_name = req["llm_name"]
         api_key = apikey_json(["yiyan_ak", "yiyan_sk"])
 
     elif factory == "Fish Audio":
-        llm_name = req["llm_name"]
         api_key = apikey_json(["fish_audio_ak", "fish_audio_refid"])
 
     elif factory == "Google Cloud":
-        llm_name = req["llm_name"]
         api_key = apikey_json(["google_project_id", "google_region", "google_service_account_key"])
 
     elif factory == "Azure-OpenAI":
-        llm_name = req["llm_name"]
         api_key = apikey_json(["api_key", "api_version"])
 
-    else:
-        llm_name = req["llm_name"]
-        api_key = req.get("api_key", "xxxxxxxxxxxxxxx")
-
     llm = {
         "tenant_id": current_user.id,
         "llm_factory": factory,
@@ -209,66 +204,74 @@ def add_llm():
     }
 
     msg = ""
+    mdl_nm = llm["llm_name"].split("___")[0]
     if llm["model_type"] == LLMType.EMBEDDING.value:
+        assert factory in EmbeddingModel, f"Embedding model from {factory} is not supported yet."
         mdl = EmbeddingModel[factory](
             key=llm['api_key'],
-            model_name=llm["llm_name"],
+            model_name=mdl_nm,
             base_url=llm["api_base"])
         try:
             arr, tc = mdl.encode(["Test if the api key is available"])
             if len(arr[0]) == 0:
                 raise Exception("Fail")
         except Exception as e:
-            msg += f"\nFail to access embedding model({llm['llm_name']})." + str(e)
+            msg += f"\nFail to access embedding model({mdl_nm})." + str(e)
     elif llm["model_type"] == LLMType.CHAT.value:
+        assert factory in ChatModel, f"Chat model from {factory} is not supported yet."
         mdl = ChatModel[factory](
             key=llm['api_key'],
-            model_name=llm["llm_name"],
+            model_name=mdl_nm,
             base_url=llm["api_base"]
         )
         try:
             m, tc = mdl.chat(None, [{"role": "user", "content": "Hello! How are you doing!"}], {
                 "temperature": 0.9})
-            if not tc:
+            if not tc and m.find("**ERROR**:") >= 0:
                 raise Exception(m)
         except Exception as e:
-            msg += f"\nFail to access model({llm['llm_name']})." + str(
+            msg += f"\nFail to access model({mdl_nm})." + str(
                 e)
     elif llm["model_type"] == LLMType.RERANK:
-        mdl = RerankModel[factory](
-            key=llm["api_key"],
-            model_name=llm["llm_name"],
-            base_url=llm["api_base"]
-        )
+        assert factory in RerankModel, f"RE-rank model from {factory} is not supported yet."
         try:
+            mdl = RerankModel[factory](
+                key=llm["api_key"],
+                model_name=mdl_nm,
+                base_url=llm["api_base"]
+            )
             arr, tc = mdl.similarity("Hello~ Ragflower!", ["Hi, there!", "Ohh, my friend!"])
             if len(arr) == 0:
                 raise Exception("Not known.")
+        except KeyError:
+            msg += f"{factory} dose not support this model({mdl_nm})"
         except Exception as e:
-            msg += f"\nFail to access model({llm['llm_name']})." + str(
+            msg += f"\nFail to access model({mdl_nm})." + str(
                 e)
     elif llm["model_type"] == LLMType.IMAGE2TEXT.value:
+        assert factory in CvModel, f"Image to text model from {factory} is not supported yet."
         mdl = CvModel[factory](
             key=llm["api_key"],
-            model_name=llm["llm_name"],
+            model_name=mdl_nm,
             base_url=llm["api_base"]
         )
         try:
             with open(os.path.join(get_project_base_directory(), "web/src/assets/yay.jpg"), "rb") as f:
                 m, tc = mdl.describe(f.read())
-                if not tc:
+                if not m and not tc:
                     raise Exception(m)
         except Exception as e:
-            msg += f"\nFail to access model({llm['llm_name']})." + str(e)
+            msg += f"\nFail to access model({mdl_nm})." + str(e)
     elif llm["model_type"] == LLMType.TTS:
+        assert factory in TTSModel, f"TTS model from {factory} is not supported yet."
         mdl = TTSModel[factory](
-            key=llm["api_key"], model_name=llm["llm_name"], base_url=llm["api_base"]
+            key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"]
         )
         try:
             for resp in mdl.tts("Hello~ Ragflower!"):
                 pass
         except RuntimeError as e:
-            msg += f"\nFail to access model({llm['llm_name']})." + str(e)
+            msg += f"\nFail to access model({mdl_nm})." + str(e)
     else:
         # TODO: check other type of models
         pass
@@ -343,8 +346,6 @@ def list_app():
 
         llm_set = set([m["llm_name"] + "@" + m["fid"] for m in llms])
         for o in objs:
-            if not o.api_key:
-                continue
             if o.llm_name + "@" + o.llm_factory in llm_set:
                 continue
             llms.append({"llm_name": o.llm_name, "model_type": o.model_type, "fid": o.llm_factory, "available": True})
@@ -359,4 +360,4 @@ def list_app():
 
         return get_json_result(data=res)
     except Exception as e:
-        return server_error_response(e)
+        return server_error_response(e)

api/apps/plugin_app.py

@@ -0,0 +1,12 @@
+from flask import Response
+from flask_login import login_required
+from api.utils.api_utils import get_json_result
+from plugin import GlobalPluginManager
+
+@manager.route('/llm_tools', methods=['GET'])  # noqa: F821
+@login_required
+def llm_tools() -> Response:
+    tools = GlobalPluginManager.get_llm_tools()
+    tools_metadata = [t.get_metadata() for t in tools]
+
+    return get_json_result(data=tools_metadata)

api/apps/sdk/agent.py

@@ -14,8 +14,14 @@
 #  limitations under the License.
 #
 
+import json
+import time
+from typing import Any, cast
 from api.db.services.canvas_service import UserCanvasService
-from api.utils.api_utils import get_error_data_result, token_required
+from api.db.services.user_canvas_version import UserCanvasVersionService
+from api.settings import RetCode
+from api.utils import get_uuid
+from api.utils.api_utils import get_data_error_result, get_error_data_result, get_json_result, token_required
 from api.utils.api_utils import get_result
 from flask import request
 
@@ -37,3 +43,86 @@ def list_agents(tenant_id):
         desc = True
     canvas = UserCanvasService.get_list(tenant_id,page_number,items_per_page,orderby,desc,id,title)
     return get_result(data=canvas)
+
+
+@manager.route("/agents", methods=["POST"])  # noqa: F821
+@token_required
+def create_agent(tenant_id: str):
+    req: dict[str, Any] = cast(dict[str, Any], request.json)
+    req["user_id"] = tenant_id
+
+    if req.get("dsl") is not None:
+        if not isinstance(req["dsl"], str):
+            req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
+
+        req["dsl"] = json.loads(req["dsl"])
+    else:
+        return get_json_result(data=False, message="No DSL data in request.", code=RetCode.ARGUMENT_ERROR)
+
+    if req.get("title") is not None:
+        req["title"] = req["title"].strip()
+    else:
+        return get_json_result(data=False, message="No title in request.", code=RetCode.ARGUMENT_ERROR)
+
+    if UserCanvasService.query(user_id=tenant_id, title=req["title"]):
+        return get_data_error_result(message=f"Agent with title {req['title']} already exists.")
+
+    agent_id = get_uuid()
+    req["id"] = agent_id
+
+    if not UserCanvasService.save(**req):
+        return get_data_error_result(message="Fail to create agent.")
+
+    UserCanvasVersionService.insert(
+        user_canvas_id=agent_id,
+        title="{0}_{1}".format(req["title"], time.strftime("%Y_%m_%d_%H_%M_%S")),
+        dsl=req["dsl"]
+    )
+
+    return get_json_result(data=True)
+
+
+@manager.route("/agents/<agent_id>", methods=["PUT"])  # noqa: F821
+@token_required
+def update_agent(tenant_id: str, agent_id: str):
+    req: dict[str, Any] = {k: v for k, v in cast(dict[str, Any], request.json).items() if v is not None}
+    req["user_id"] = tenant_id
+
+    if req.get("dsl") is not None:
+        if not isinstance(req["dsl"], str):
+            req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
+
+        req["dsl"] = json.loads(req["dsl"])
+    
+    if req.get("title") is not None:
+        req["title"] = req["title"].strip()
+
+    if not UserCanvasService.query(user_id=tenant_id, id=agent_id):
+        return get_json_result(
+            data=False, message="Only owner of canvas authorized for this operation.",
+            code=RetCode.OPERATING_ERROR)
+
+    UserCanvasService.update_by_id(agent_id, req)
+
+    if req.get("dsl") is not None:
+        UserCanvasVersionService.insert(
+            user_canvas_id=agent_id,
+            title="{0}_{1}".format(req["title"], time.strftime("%Y_%m_%d_%H_%M_%S")),
+            dsl=req["dsl"]
+        )
+
+        UserCanvasVersionService.delete_all_versions(agent_id)
+
+    return get_json_result(data=True)
+
+
+@manager.route("/agents/<agent_id>", methods=["DELETE"])  # noqa: F821
+@token_required
+def delete_agent(tenant_id: str, agent_id: str):
+    if not UserCanvasService.query(user_id=tenant_id, id=agent_id):
+        return get_json_result(
+            data=False, message="Only owner of canvas authorized for this operation.",
+            code=RetCode.OPERATING_ERROR)
+
+    UserCanvasService.delete_by_id(agent_id)
+    return get_json_result(data=True)

api/apps/sdk/chat.py

@@ -13,7 +13,10 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
+
 from flask import request
+
 from api import settings
 from api.db import StatusEnum
 from api.db.services.dialog_service import DialogService
@@ -21,17 +24,14 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import TenantLLMService
 from api.db.services.user_service import TenantService
 from api.utils import get_uuid
-from api.utils.api_utils import get_error_data_result, token_required
-from api.utils.api_utils import get_result
+from api.utils.api_utils import check_duplicate_ids, get_error_data_result, get_result, token_required
 
 
-@manager.route('/chats', methods=['POST'])  # noqa: F821
+@manager.route("/chats", methods=["POST"])  # noqa: F821
 @token_required
 def create(tenant_id):
     req = request.json
-    ids = req.get("dataset_ids")
-    if not ids:
-        return get_error_data_result(message="`dataset_ids` is required")
+    ids = [i for i in req.get("dataset_ids", []) if i]
     for kb_id in ids:
         kbs = KnowledgebaseService.accessible(kb_id=kb_id, user_id=tenant_id)
         if not kbs:
@@ -40,32 +40,30 @@ def create(tenant_id):
         kb = kbs[0]
         if kb.chunk_num == 0:
             return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
-    kbs = KnowledgebaseService.get_by_ids(ids)
-    embd_count = list(set([kb.embd_id for kb in kbs]))
-    if len(embd_count) != 1:
-        return get_result(message='Datasets use different embedding models."',
-                          code=settings.RetCode.AUTHENTICATION_ERROR)
+
+    kbs = KnowledgebaseService.get_by_ids(ids) if ids else []
+    embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]  # remove vendor suffix for comparison
+    embd_count = list(set(embd_ids))
+    if len(embd_count) > 1:
+        return get_result(message='Datasets use different embedding models."', code=settings.RetCode.AUTHENTICATION_ERROR)
     req["kb_ids"] = ids
     # llm
     llm = req.get("llm")
     if llm:
         if "model_name" in llm:
             req["llm_id"] = llm.pop("model_name")
-            if not TenantLLMService.query(tenant_id=tenant_id, llm_name=req["llm_id"], model_type="chat"):
-                return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
+            if req.get("llm_id") is not None:
+                llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(req["llm_id"])
+                if not TenantLLMService.query(tenant_id=tenant_id, llm_name=llm_name, llm_factory=llm_factory, model_type="chat"):
+                    return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
         req["llm_setting"] = req.pop("llm")
     e, tenant = TenantService.get_by_id(tenant_id)
     if not e:
         return get_error_data_result(message="Tenant not found!")
     # prompt
     prompt = req.get("prompt")
-    key_mapping = {"parameters": "variables",
-                   "prologue": "opener",
-                   "quote": "show_quote",
-                   "system": "prompt",
-                   "rerank_id": "rerank_model",
-                   "vector_similarity_weight": "keywords_similarity_weight"}
-    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id","top_k"]
+    key_mapping = {"parameters": "variables", "prologue": "opener", "quote": "show_quote", "system": "prompt", "rerank_id": "rerank_model", "vector_similarity_weight": "keywords_similarity_weight"}
+    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id", "top_k"]
     if prompt:
         for new_key, old_key in key_mapping.items():
             if old_key in prompt:
@@ -83,9 +81,7 @@ def create(tenant_id):
     req["rerank_id"] = req.get("rerank_id", "")
     if req.get("rerank_id"):
         value_rerank_model = ["BAAI/bge-reranker-v2-m3", "maidalun1020/bce-reranker-base_v1"]
-        if req["rerank_id"] not in value_rerank_model and not TenantLLMService.query(tenant_id=tenant_id,
-                                                                                     llm_name=req.get("rerank_id"),
-                                                                                     model_type="rerank"):
+        if req["rerank_id"] not in value_rerank_model and not TenantLLMService.query(tenant_id=tenant_id, llm_name=req.get("rerank_id"), model_type="rerank"):
             return get_error_data_result(f"`rerank_model` {req.get('rerank_id')} doesn't exist")
     if not req.get("llm_id"):
         req["llm_id"] = tenant.llm_id
@@ -104,27 +100,24 @@ def create(tenant_id):
       {knowledge}
       The above is the knowledge base.""",
         "prologue": "Hi! I'm your assistant, what can I do for you?",
-        "parameters": [
-            {"key": "knowledge", "optional": False}
-        ],
+        "parameters": [{"key": "knowledge", "optional": False}],
         "empty_response": "Sorry! No relevant content was found in the knowledge base!",
         "quote": True,
         "tts": False,
-        "refine_multiturn": True
+        "refine_multiturn": True,
     }
     key_list_2 = ["system", "prologue", "parameters", "empty_response", "quote", "tts", "refine_multiturn"]
     if "prompt_config" not in req:
-        req['prompt_config'] = {}
+        req["prompt_config"] = {}
     for key in key_list_2:
-        temp = req['prompt_config'].get(key)
-        if (not temp and key == 'system') or (key not in req["prompt_config"]):
-            req['prompt_config'][key] = default_prompt[key]
-    for p in req['prompt_config']["parameters"]:
+        temp = req["prompt_config"].get(key)
+        if (not temp and key == "system") or (key not in req["prompt_config"]):
+            req["prompt_config"][key] = default_prompt[key]
+    for p in req["prompt_config"]["parameters"]:
         if p["optional"]:
             continue
-        if req['prompt_config']["system"].find("{%s}" % p["key"]) < 0:
-            return get_error_data_result(
-                message="Parameter '{}' is not used".format(p["key"]))
+        if req["prompt_config"]["system"].find("{%s}" % p["key"]) < 0:
+            return get_error_data_result(message="Parameter '{}' is not used".format(p["key"]))
     # save
     if not DialogService.save(**req):
         return get_error_data_result(message="Fail to new a chat!")
@@ -139,10 +132,7 @@ def create(tenant_id):
         renamed_dict[new_key] = value
     res["prompt"] = renamed_dict
     del res["prompt_config"]
-    new_dict = {"similarity_threshold": res["similarity_threshold"],
-                "keywords_similarity_weight": 1-res["vector_similarity_weight"],
-                "top_n": res["top_n"],
-                "rerank_model": res['rerank_id']}
+    new_dict = {"similarity_threshold": res["similarity_threshold"], "keywords_similarity_weight": 1 - res["vector_similarity_weight"], "top_n": res["top_n"], "rerank_model": res["rerank_id"]}
     res["prompt"].update(new_dict)
     for key in key_list:
         del res[key]
@@ -154,53 +144,47 @@ def create(tenant_id):
     return get_result(data=res)
 
 
-@manager.route('/chats/<chat_id>', methods=['PUT'])  # noqa: F821
+@manager.route("/chats/<chat_id>", methods=["PUT"])  # noqa: F821
 @token_required
 def update(tenant_id, chat_id):
     if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
-        return get_error_data_result(message='You do not own the chat')
+        return get_error_data_result(message="You do not own the chat")
     req = request.json
     ids = req.get("dataset_ids")
     if "show_quotation" in req:
         req["do_refer"] = req.pop("show_quotation")
-    if "dataset_ids" in req:
-        if not ids:
-            return get_error_data_result("`dataset_ids` can't be empty")
-        if ids:
-            for kb_id in ids:
-                kbs = KnowledgebaseService.accessible(kb_id=kb_id, user_id=tenant_id)
-                if not kbs:
-                    return get_error_data_result(f"You don't own the dataset {kb_id}")
-                kbs = KnowledgebaseService.query(id=kb_id)
-                kb = kbs[0]
-                if kb.chunk_num == 0:
-                    return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
-            kbs = KnowledgebaseService.get_by_ids(ids)
-            embd_count = list(set([kb.embd_id for kb in kbs]))
-            if len(embd_count) != 1:
-                return get_result(
-                    message='Datasets use different embedding models."',
-                    code=settings.RetCode.AUTHENTICATION_ERROR)
-            req["kb_ids"] = ids
+    if ids is not None:
+        for kb_id in ids:
+            kbs = KnowledgebaseService.accessible(kb_id=kb_id, user_id=tenant_id)
+            if not kbs:
+                return get_error_data_result(f"You don't own the dataset {kb_id}")
+            kbs = KnowledgebaseService.query(id=kb_id)
+            kb = kbs[0]
+            if kb.chunk_num == 0:
+                return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
+
+        kbs = KnowledgebaseService.get_by_ids(ids)
+        embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]  # remove vendor suffix for comparison
+        embd_count = list(set(embd_ids))
+        if len(embd_count) != 1:
+            return get_result(message='Datasets use different embedding models."', code=settings.RetCode.AUTHENTICATION_ERROR)
+        req["kb_ids"] = ids
     llm = req.get("llm")
     if llm:
         if "model_name" in llm:
             req["llm_id"] = llm.pop("model_name")
-            if not TenantLLMService.query(tenant_id=tenant_id, llm_name=req["llm_id"], model_type="chat"):
-                return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
+            if req.get("llm_id") is not None:
+                llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(req["llm_id"])
+                if not TenantLLMService.query(tenant_id=tenant_id, llm_name=llm_name, llm_factory=llm_factory, model_type="chat"):
+                    return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
         req["llm_setting"] = req.pop("llm")
     e, tenant = TenantService.get_by_id(tenant_id)
     if not e:
         return get_error_data_result(message="Tenant not found!")
     # prompt
     prompt = req.get("prompt")
-    key_mapping = {"parameters": "variables",
-                   "prologue": "opener",
-                   "quote": "show_quote",
-                   "system": "prompt",
-                   "rerank_id": "rerank_model",
-                   "vector_similarity_weight": "keywords_similarity_weight"}
-    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id","top_k"]
+    key_mapping = {"parameters": "variables", "prologue": "opener", "quote": "show_quote", "system": "prompt", "rerank_id": "rerank_model", "vector_similarity_weight": "keywords_similarity_weight"}
+    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id", "top_k"]
     if prompt:
         for new_key, old_key in key_mapping.items():
             if old_key in prompt:
@@ -213,17 +197,13 @@ def update(tenant_id, chat_id):
     res = res.to_json()
     if req.get("rerank_id"):
         value_rerank_model = ["BAAI/bge-reranker-v2-m3", "maidalun1020/bce-reranker-base_v1"]
-        if req["rerank_id"] not in value_rerank_model and not TenantLLMService.query(tenant_id=tenant_id,
-                                                                                     llm_name=req.get("rerank_id"),
-                                                                                     model_type="rerank"):
+        if req["rerank_id"] not in value_rerank_model and not TenantLLMService.query(tenant_id=tenant_id, llm_name=req.get("rerank_id"), model_type="rerank"):
             return get_error_data_result(f"`rerank_model` {req.get('rerank_id')} doesn't exist")
     if "name" in req:
         if not req.get("name"):
-            return get_error_data_result(message="`name` is not empty.")
-        if req["name"].lower() != res["name"].lower() \
-                and len(
-            DialogService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) > 0:
-            return get_error_data_result(message="Duplicated chat name in updating dataset.")
+            return get_error_data_result(message="`name` cannot be empty.")
+        if req["name"].lower() != res["name"].lower() and len(DialogService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) > 0:
+            return get_error_data_result(message="Duplicated chat name in updating chat.")
     if "prompt_config" in req:
         res["prompt_config"].update(req["prompt_config"])
         for p in res["prompt_config"]["parameters"]:
@@ -245,9 +225,11 @@ def update(tenant_id, chat_id):
     return get_result()
 
 
-@manager.route('/chats', methods=['DELETE'])  # noqa: F821
+@manager.route("/chats", methods=["DELETE"])  # noqa: F821
 @token_required
 def delete(tenant_id):
+    errors = []
+    success_count = 0
     req = request.json
     if not req:
         ids = None
@@ -260,15 +242,33 @@ def delete(tenant_id):
             id_list.append(dia.id)
     else:
         id_list = ids
-    for id in id_list:
+
+    unique_id_list, duplicate_messages = check_duplicate_ids(id_list, "assistant")
+
+    for id in unique_id_list:
         if not DialogService.query(tenant_id=tenant_id, id=id, status=StatusEnum.VALID.value):
-            return get_error_data_result(message=f"You don't own the chat {id}")
+            errors.append(f"Assistant({id}) not found.")
+            continue
         temp_dict = {"status": StatusEnum.INVALID.value}
         DialogService.update_by_id(id, temp_dict)
+        success_count += 1
+
+    if errors:
+        if success_count > 0:
+            return get_result(data={"success_count": success_count, "errors": errors}, message=f"Partially deleted {success_count} chats with {len(errors)} errors")
+        else:
+            return get_error_data_result(message="; ".join(errors))
+
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(message=f"Partially deleted {success_count} chats with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages})
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+
     return get_result()
 
 
-@manager.route('/chats', methods=['GET'])  # noqa: F821
+@manager.route("/chats", methods=["GET"])  # noqa: F821
 @token_required
 def list_chat(tenant_id):
     id = request.args.get("id")
@@ -288,13 +288,15 @@ def list_chat(tenant_id):
     if not chats:
         return get_result(data=[])
     list_assts = []
-    key_mapping = {"parameters": "variables",
-                   "prologue": "opener",
-                   "quote": "show_quote",
-                   "system": "prompt",
-                   "rerank_id": "rerank_model",
-                   "vector_similarity_weight": "keywords_similarity_weight",
-                   "do_refer": "show_quotation"}
+    key_mapping = {
+        "parameters": "variables",
+        "prologue": "opener",
+        "quote": "show_quote",
+        "system": "prompt",
+        "rerank_id": "rerank_model",
+        "vector_similarity_weight": "keywords_similarity_weight",
+        "do_refer": "show_quotation",
+    }
     key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
     for res in chats:
         renamed_dict = {}
@@ -303,10 +305,7 @@ def list_chat(tenant_id):
             renamed_dict[new_key] = value
         res["prompt"] = renamed_dict
         del res["prompt_config"]
-        new_dict = {"similarity_threshold": res["similarity_threshold"],
-                    "keywords_similarity_weight": 1-res["vector_similarity_weight"],
-                    "top_n": res["top_n"],
-                    "rerank_model": res['rerank_id']}
+        new_dict = {"similarity_threshold": res["similarity_threshold"], "keywords_similarity_weight": 1 - res["vector_similarity_weight"], "top_n": res["top_n"], "rerank_model": res["rerank_id"]}
         res["prompt"].update(new_dict)
         for key in key_list:
             del res[key]
@@ -316,7 +315,8 @@ def list_chat(tenant_id):
         for kb_id in res["kb_ids"]:
             kb = KnowledgebaseService.query(id=kb_id)
             if not kb:
-                return get_error_data_result(message=f"Don't exist the kb {kb_id}")
+                logging.warning(f"The kb {kb_id} does not exist.")
+                continue
             kb_list.append(kb[0].to_json())
         del res["kb_ids"]
         res["datasets"] = kb_list

api/apps/sdk/dataset.py

@@ -14,24 +14,44 @@
 #  limitations under the License.
 #
 
+
+import logging
+import os
+
 from flask import request
-from api.db import StatusEnum, FileSource
+from peewee import OperationalError
+
+from api import settings
+from api.db import FileSource, StatusEnum
 from api.db.db_models import File
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.llm_service import TenantLLMService, LLMService
 from api.db.services.user_service import TenantService
-from api import settings
 from api.utils import get_uuid
 from api.utils.api_utils import (
-    get_result,
-    token_required,
+    deep_merge,
+    get_error_argument_result,
     get_error_data_result,
-    valid,
+    get_error_operating_result,
+    get_error_permission_result,
     get_parser_config,
+    get_result,
+    remap_dictionary_keys,
+    token_required,
+    verify_embedding_availability,
 )
+from api.utils.validation_utils import (
+    CreateDatasetReq,
+    DeleteDatasetReq,
+    ListDatasetReq,
+    UpdateDatasetReq,
+    validate_and_parse_json_request,
+    validate_and_parse_request_args,
+)
+from rag.nlp import search
+from rag.settings import PAGERANK_FLD
 
 
 @manager.route("/datasets", methods=["POST"])  # noqa: F821
@@ -62,18 +82,23 @@ def create(tenant_id):
             name:
               type: string
               description: Name of the dataset.
+            avatar:
+              type: string
+              description: Base64 encoding of the avatar.
+            description:
+              type: string
+              description: Description of the dataset.
+            embedding_model:
+              type: string
+              description: Embedding model Name.
             permission:
               type: string
               enum: ['me', 'team']
               description: Dataset permission.
-            language:
-              type: string
-              enum: ['Chinese', 'English']
-              description: Language of the dataset.
             chunk_method:
               type: string
-              enum: ["naive", "manual", "qa", "table", "paper", "book", "laws",
-                     "presentation", "picture", "one", "knowledge_graph", "email", "tag"
+              enum: ["naive", "book", "email", "laws", "manual", "one", "paper",
+                     "picture", "presentation", "qa", "table", "tag"
                      ]
               description: Chunking method.
             parser_config:
@@ -88,108 +113,47 @@ def create(tenant_id):
             data:
               type: object
     """
-    req = request.json
-    e, t = TenantService.get_by_id(tenant_id)
-    permission = req.get("permission")
-    language = req.get("language")
-    chunk_method = req.get("chunk_method")
-    parser_config = req.get("parser_config")
-    valid_permission = ["me", "team"]
-    valid_language = ["Chinese", "English"]
-    valid_chunk_method = [
-        "naive",
-        "manual",
-        "qa",
-        "table",
-        "paper",
-        "book",
-        "laws",
-        "presentation",
-        "picture",
-        "one",
-        "knowledge_graph",
-        "email",
-        "tag"
-    ]
-    check_validation = valid(
-        permission,
-        valid_permission,
-        language,
-        valid_language,
-        chunk_method,
-        valid_chunk_method,
-    )
-    if check_validation:
-        return check_validation
-    req["parser_config"] = get_parser_config(chunk_method, parser_config)
-    if "tenant_id" in req:
-        return get_error_data_result(message="`tenant_id` must not be provided")
-    if "chunk_count" in req or "document_count" in req:
-        return get_error_data_result(
-            message="`chunk_count` or `document_count` must not be provided"
-        )
-    if "name" not in req:
-        return get_error_data_result(message="`name` is not empty!")
-    req["id"] = get_uuid()
-    req["name"] = req["name"].strip()
-    if req["name"] == "":
-        return get_error_data_result(message="`name` is not empty string!")
-    if KnowledgebaseService.query(
-        name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value
-    ):
-        return get_error_data_result(
-            message="Duplicated dataset name in creating dataset."
-        )
-    req["tenant_id"] = req["created_by"] = tenant_id
-    if not req.get("embedding_model"):
-        req["embedding_model"] = t.embd_id
-    else:
-        valid_embedding_models = [
-            "BAAI/bge-large-zh-v1.5",
-            "BAAI/bge-base-en-v1.5",
-            "BAAI/bge-large-en-v1.5",
-            "BAAI/bge-small-en-v1.5",
-            "BAAI/bge-small-zh-v1.5",
-            "jinaai/jina-embeddings-v2-base-en",
-            "jinaai/jina-embeddings-v2-small-en",
-            "nomic-ai/nomic-embed-text-v1.5",
-            "sentence-transformers/all-MiniLM-L6-v2",
-            "text-embedding-v2",
-            "text-embedding-v3",
-            "maidalun1020/bce-embedding-base_v1",
-        ]
-        embd_model = LLMService.query(
-            llm_name=req["embedding_model"], model_type="embedding"
-        )
-        if embd_model:
-            if req["embedding_model"] not in valid_embedding_models and not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding",llm_name=req.get("embedding_model"),):
-                return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
-        if not embd_model:
-            embd_model=TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model"))
-        if not embd_model:
-            return get_error_data_result(
-                f"`embedding_model` {req.get('embedding_model')} doesn't exist"
-            )
-    key_mapping = {
-        "chunk_num": "chunk_count",
-        "doc_num": "document_count",
-        "parser_id": "chunk_method",
-        "embd_id": "embedding_model",
-    }
-    mapped_keys = {
-        new_key: req[old_key]
-        for new_key, old_key in key_mapping.items()
-        if old_key in req
-    }
-    req.update(mapped_keys)
-    if not KnowledgebaseService.save(**req):
-        return get_error_data_result(message="Create dataset error.(Database error)")
-    renamed_data = {}
-    e, k = KnowledgebaseService.get_by_id(req["id"])
-    for key, value in k.to_dict().items():
-        new_key = key_mapping.get(key, key)
-        renamed_data[new_key] = value
-    return get_result(data=renamed_data)
+    # Field name transformations during model dump:
+    # | Original       | Dump Output  |
+    # |----------------|-------------|
+    # | embedding_model| embd_id     |
+    # | chunk_method   | parser_id   |
+    req, err = validate_and_parse_json_request(request, CreateDatasetReq)
+    if err is not None:
+        return get_error_argument_result(err)
+
+    try:
+        if KnowledgebaseService.get_or_none(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
+            return get_error_operating_result(message=f"Dataset name '{req['name']}' already exists")
+
+        req["parser_config"] = get_parser_config(req["parser_id"], req["parser_config"])
+        req["id"] = get_uuid()
+        req["tenant_id"] = tenant_id
+        req["created_by"] = tenant_id
+
+        ok, t = TenantService.get_by_id(tenant_id)
+        if not ok:
+            return get_error_permission_result(message="Tenant not found")
+
+        if not req.get("embd_id"):
+            req["embd_id"] = t.embd_id
+        else:
+            ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
+            if not ok:
+                return err
+
+        if not KnowledgebaseService.save(**req):
+            return get_error_data_result(message="Create dataset error.(Database error)")
+
+        ok, k = KnowledgebaseService.get_by_id(req["id"])
+        if not ok:
+            return get_error_data_result(message="Dataset created failed")
+
+        response_data = remap_dictionary_keys(k.to_dict())
+        return get_result(data=response_data)
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
 
 
 @manager.route("/datasets", methods=["DELETE"])  # noqa: F821
@@ -214,52 +178,78 @@ def delete(tenant_id):
         required: true
         schema:
           type: object
+          required:
+            - ids
           properties:
             ids:
-              type: array
+              type: array or null
               items:
                 type: string
-              description: List of dataset IDs to delete.
+              description: |
+                Specifies the datasets to delete:
+                - If `null`, all datasets will be deleted.
+                - If an array of IDs, only the specified datasets will be deleted.
+                - If an empty array, no datasets will be deleted.
     responses:
       200:
         description: Successful operation.
         schema:
           type: object
     """
-    req = request.json
-    if not req:
-        ids = None
-    else:
-        ids = req.get("ids")
-    if not ids:
-        id_list = []
-        kbs = KnowledgebaseService.query(tenant_id=tenant_id)
-        for kb in kbs:
-            id_list.append(kb.id)
-    else:
-        id_list = ids
-    for id in id_list:
-        kbs = KnowledgebaseService.query(id=id, tenant_id=tenant_id)
-        if not kbs:
-            return get_error_data_result(message=f"You don't own the dataset {id}")
-        for doc in DocumentService.query(kb_id=id):
-            if not DocumentService.remove_document(doc, tenant_id):
-                return get_error_data_result(
-                    message="Remove document error.(Database error)"
+    req, err = validate_and_parse_json_request(request, DeleteDatasetReq)
+    if err is not None:
+        return get_error_argument_result(err)
+
+    try:
+        kb_id_instance_pairs = []
+        if req["ids"] is None:
+            kbs = KnowledgebaseService.query(tenant_id=tenant_id)
+            for kb in kbs:
+                kb_id_instance_pairs.append((kb.id, kb))
+
+        else:
+            error_kb_ids = []
+            for kb_id in req["ids"]:
+                kb = KnowledgebaseService.get_or_none(id=kb_id, tenant_id=tenant_id)
+                if kb is None:
+                    error_kb_ids.append(kb_id)
+                    continue
+                kb_id_instance_pairs.append((kb_id, kb))
+            if len(error_kb_ids) > 0:
+                return get_error_permission_result(message=f"""User '{tenant_id}' lacks permission for datasets: '{", ".join(error_kb_ids)}'""")
+
+        errors = []
+        success_count = 0
+        for kb_id, kb in kb_id_instance_pairs:
+            for doc in DocumentService.query(kb_id=kb_id):
+                if not DocumentService.remove_document(doc, tenant_id):
+                    errors.append(f"Remove document '{doc.id}' error for dataset '{kb_id}'")
+                    continue
+                f2d = File2DocumentService.get_by_document_id(doc.id)
+                FileService.filter_delete(
+                    [
+                        File.source_type == FileSource.KNOWLEDGEBASE,
+                        File.id == f2d[0].file_id,
+                    ]
                 )
-            f2d = File2DocumentService.get_by_document_id(doc.id)
-            FileService.filter_delete(
-                [
-                    File.source_type == FileSource.KNOWLEDGEBASE,
-                    File.id == f2d[0].file_id,
-                ]
-            )
-            File2DocumentService.delete_by_document_id(doc.id)
-        FileService.filter_delete(
-            [File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
-        if not KnowledgebaseService.delete_by_id(id):
-            return get_error_data_result(message="Delete dataset error.(Database error)")
-    return get_result(code=settings.RetCode.SUCCESS)
+                File2DocumentService.delete_by_document_id(doc.id)
+            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kb.name])
+            if not KnowledgebaseService.delete_by_id(kb_id):
+                errors.append(f"Delete dataset error for {kb_id}")
+                continue
+            success_count += 1
+
+        if not errors:
+            return get_result()
+
+        error_message = f"Successfully deleted {success_count} datasets, {len(errors)} failed. Details: {'; '.join(errors)[:128]}..."
+        if success_count == 0:
+            return get_error_data_result(message=error_message)
+
+        return get_result(data={"success_count": success_count, "errors": errors[:5]}, message=error_message)
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
 
 
 @manager.route("/datasets/<dataset_id>", methods=["PUT"])  # noqa: F821
@@ -293,20 +283,28 @@ def update(tenant_id, dataset_id):
             name:
               type: string
               description: New name of the dataset.
+            avatar:
+              type: string
+              description: Updated base64 encoding of the avatar.
+            description:
+              type: string
+              description: Updated description of the dataset.
+            embedding_model:
+              type: string
+              description: Updated embedding model Name.
             permission:
               type: string
               enum: ['me', 'team']
-              description: Updated permission.
-            language:
-              type: string
-              enum: ['Chinese', 'English']
-              description: Updated language.
+              description: Updated dataset permission.
             chunk_method:
               type: string
-              enum: ["naive", "manual", "qa", "table", "paper", "book", "laws",
-                     "presentation", "picture", "one", "knowledge_graph", "email", "tag"
+              enum: ["naive", "book", "email", "laws", "manual", "one", "paper",
+                     "picture", "presentation", "qa", "table", "tag"
                      ]
               description: Updated chunking method.
+            pagerank:
+              type: integer
+              description: Updated page rank.
             parser_config:
               type: object
               description: Updated parser configuration.
@@ -316,126 +314,72 @@ def update(tenant_id, dataset_id):
         schema:
           type: object
     """
-    if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
-        return get_error_data_result(message="You don't own the dataset")
-    req = request.json
-    e, t = TenantService.get_by_id(tenant_id)
-    invalid_keys = {"id", "embd_id", "chunk_num", "doc_num", "parser_id"}
-    if any(key in req for key in invalid_keys):
-        return get_error_data_result(message="The input parameters are invalid.")
-    permission = req.get("permission")
-    language = req.get("language")
-    chunk_method = req.get("chunk_method")
-    parser_config = req.get("parser_config")
-    valid_permission = ["me", "team"]
-    valid_language = ["Chinese", "English"]
-    valid_chunk_method = [
-        "naive",
-        "manual",
-        "qa",
-        "table",
-        "paper",
-        "book",
-        "laws",
-        "presentation",
-        "picture",
-        "one",
-        "knowledge_graph",
-        "email",
-        "tag"
-    ]
-    check_validation = valid(
-        permission,
-        valid_permission,
-        language,
-        valid_language,
-        chunk_method,
-        valid_chunk_method,
-    )
-    if check_validation:
-        return check_validation
-    if "tenant_id" in req:
-        if req["tenant_id"] != tenant_id:
-            return get_error_data_result(message="Can't change `tenant_id`.")
-    e, kb = KnowledgebaseService.get_by_id(dataset_id)
-    if "parser_config" in req:
-        temp_dict = kb.parser_config
-        temp_dict.update(req["parser_config"])
-        req["parser_config"] = temp_dict
-    if "chunk_count" in req:
-        if req["chunk_count"] != kb.chunk_num:
-            return get_error_data_result(message="Can't change `chunk_count`.")
-        req.pop("chunk_count")
-    if "document_count" in req:
-        if req["document_count"] != kb.doc_num:
-            return get_error_data_result(message="Can't change `document_count`.")
-        req.pop("document_count")
-    if "chunk_method" in req:
-        if kb.chunk_num != 0 and req["chunk_method"] != kb.parser_id:
-            return get_error_data_result(
-                message="If `chunk_count` is not 0, `chunk_method` is not changeable."
-            )
-        req["parser_id"] = req.pop("chunk_method")
-        if req["parser_id"] != kb.parser_id:
-            if not req.get("parser_config"):
-                req["parser_config"] = get_parser_config(chunk_method, parser_config)
-    if "embedding_model" in req:
-        if kb.chunk_num != 0 and req["embedding_model"] != kb.embd_id:
-            return get_error_data_result(
-                message="If `chunk_count` is not 0, `embedding_model` is not changeable."
-            )
-        if not req.get("embedding_model"):
-            return get_error_data_result("`embedding_model` can't be empty")
-        valid_embedding_models = [
-            "BAAI/bge-large-zh-v1.5",
-            "BAAI/bge-base-en-v1.5",
-            "BAAI/bge-large-en-v1.5",
-            "BAAI/bge-small-en-v1.5",
-            "BAAI/bge-small-zh-v1.5",
-            "jinaai/jina-embeddings-v2-base-en",
-            "jinaai/jina-embeddings-v2-small-en",
-            "nomic-ai/nomic-embed-text-v1.5",
-            "sentence-transformers/all-MiniLM-L6-v2",
-            "text-embedding-v2",
-            "text-embedding-v3",
-            "maidalun1020/bce-embedding-base_v1",
-        ]
-        embd_model = LLMService.query(
-            llm_name=req["embedding_model"], model_type="embedding"
-        )
-        if embd_model:
-            if req["embedding_model"] not in valid_embedding_models and not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding",llm_name=req.get("embedding_model"),):
-                return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
-        if not embd_model:
-            embd_model=TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model"))
+    # Field name transformations during model dump:
+    # | Original       | Dump Output  |
+    # |----------------|-------------|
+    # | embedding_model| embd_id     |
+    # | chunk_method   | parser_id   |
+    extras = {"dataset_id": dataset_id}
+    req, err = validate_and_parse_json_request(request, UpdateDatasetReq, extras=extras, exclude_unset=True)
+    if err is not None:
+        return get_error_argument_result(err)
 
-        if not embd_model:
-            return get_error_data_result(
-                f"`embedding_model` {req.get('embedding_model')} doesn't exist"
-            )
-        req["embd_id"] = req.pop("embedding_model")
-    if "name" in req:
-        req["name"] = req["name"].strip()
-        if (
-            req["name"].lower() != kb.name.lower()
-            and len(
-                KnowledgebaseService.query(
-                    name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value
-                )
-            )
-            > 0
-        ):
-            return get_error_data_result(
-                message="Duplicated dataset name in updating dataset."
-            )
-    if not KnowledgebaseService.update_by_id(kb.id, req):
-        return get_error_data_result(message="Update dataset error.(Database error)")
-    return get_result(code=settings.RetCode.SUCCESS)
+    if not req:
+        return get_error_argument_result(message="No properties were modified")
+
+    try:
+        kb = KnowledgebaseService.get_or_none(id=dataset_id, tenant_id=tenant_id)
+        if kb is None:
+            return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{dataset_id}'")
+
+        if req.get("parser_config"):
+            req["parser_config"] = deep_merge(kb.parser_config, req["parser_config"])
+
+        if (chunk_method := req.get("parser_id")) and chunk_method != kb.parser_id:
+            if not req.get("parser_config"):
+                req["parser_config"] = get_parser_config(chunk_method, None)
+        elif "parser_config" in req and not req["parser_config"]:
+            del req["parser_config"]
+
+        if "name" in req and req["name"].lower() != kb.name.lower():
+            exists = KnowledgebaseService.get_or_none(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)
+            if exists:
+                return get_error_data_result(message=f"Dataset name '{req['name']}' already exists")
+
+        if "embd_id" in req:
+            if kb.chunk_num != 0 and req["embd_id"] != kb.embd_id:
+                return get_error_data_result(message=f"When chunk_num ({kb.chunk_num}) > 0, embedding_model must remain {kb.embd_id}")
+            ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
+            if not ok:
+                return err
+
+        if "pagerank" in req and req["pagerank"] != kb.pagerank:
+            if os.environ.get("DOC_ENGINE", "elasticsearch") == "infinity":
+                return get_error_argument_result(message="'pagerank' can only be set when doc_engine is elasticsearch")
+
+            if req["pagerank"] > 0:
+                settings.docStoreConn.update({"kb_id": kb.id}, {PAGERANK_FLD: req["pagerank"]}, search.index_name(kb.tenant_id), kb.id)
+            else:
+                # Elasticsearch requires PAGERANK_FLD be non-zero!
+                settings.docStoreConn.update({"exists": PAGERANK_FLD}, {"remove": PAGERANK_FLD}, search.index_name(kb.tenant_id), kb.id)
+
+        if not KnowledgebaseService.update_by_id(kb.id, req):
+            return get_error_data_result(message="Update dataset error.(Database error)")
+
+        ok, k = KnowledgebaseService.get_by_id(kb.id)
+        if not ok:
+            return get_error_data_result(message="Dataset created failed")
+
+        response_data = remap_dictionary_keys(k.to_dict())
+        return get_result(data=response_data)
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
 
 
 @manager.route("/datasets", methods=["GET"])  # noqa: F821
 @token_required
-def list(tenant_id):
+def list_datasets(tenant_id):
     """
     List datasets.
     ---
@@ -464,7 +408,7 @@ def list(tenant_id):
         name: page_size
         type: integer
         required: false
-        default: 1024
+        default: 30
         description: Number of items per page.
       - in: query
         name: orderby
@@ -491,45 +435,39 @@ def list(tenant_id):
           items:
             type: object
     """
-    id = request.args.get("id")
-    name = request.args.get("name")
-    if id:
-        kbs = KnowledgebaseService.get_kb_by_id(id,tenant_id)
-        if not kbs:
-            return get_error_data_result(f"You don't own the dataset {id}")
-    if name:
-        kbs = KnowledgebaseService.get_kb_by_name(name,tenant_id)
-        if not kbs:
-            return get_error_data_result(f"You don't own the dataset {name}")
-    page_number = int(request.args.get("page", 1))
-    items_per_page = int(request.args.get("page_size", 30))
-    orderby = request.args.get("orderby", "create_time")
-    if request.args.get("desc") == "False" or request.args.get("desc") == "false":
-        desc = False
-    else:
-        desc = True
-    tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)
-    kbs = KnowledgebaseService.get_list(
-        [m["tenant_id"] for m in tenants],
-        tenant_id,
-        page_number,
-        items_per_page,
-        orderby,
-        desc,
-        id,
-        name,
-    )
-    renamed_list = []
-    for kb in kbs:
-        key_mapping = {
-            "chunk_num": "chunk_count",
-            "doc_num": "document_count",
-            "parser_id": "chunk_method",
-            "embd_id": "embedding_model",
-        }
-        renamed_data = {}
-        for key, value in kb.items():
-            new_key = key_mapping.get(key, key)
-            renamed_data[new_key] = value
-        renamed_list.append(renamed_data)
-    return get_result(data=renamed_list)
+    args, err = validate_and_parse_request_args(request, ListDatasetReq)
+    if err is not None:
+        return get_error_argument_result(err)
+
+    try:
+        kb_id = request.args.get("id")
+        name = args.get("name")
+        if kb_id:
+            kbs = KnowledgebaseService.get_kb_by_id(kb_id, tenant_id)
+
+            if not kbs:
+                return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{kb_id}'")
+        if name:
+            kbs = KnowledgebaseService.get_kb_by_name(name, tenant_id)
+            if not kbs:
+                return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{name}'")
+
+        tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)
+        kbs = KnowledgebaseService.get_list(
+            [m["tenant_id"] for m in tenants],
+            tenant_id,
+            args["page"],
+            args["page_size"],
+            args["orderby"],
+            args["desc"],
+            kb_id,
+            name,
+        )
+
+        response_data_list = []
+        for kb in kbs:
+            response_data_list.append(remap_dictionary_keys(kb))
+        return get_result(data=response_data_list)
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")

api/apps/sdk/dify_retrieval.py

@@ -16,11 +16,12 @@
 from flask import request, jsonify
 
 from api.db import LLMType
-from api.db.services.dialog_service import label_question
+from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api import settings
 from api.utils.api_utils import validate_request, build_error_result, apikey_required
+from rag.app.tag import label_question
 
 
 @manager.route('/dify/retrieval', methods=['POST'])  # noqa: F821
@@ -70,12 +71,13 @@ def retrieval(tenant_id):
 
         records = []
         for c in ranks["chunks"]:
+            e, doc = DocumentService.get_by_id( c["doc_id"])
             c.pop("vector", None)
             records.append({
                 "content": c["content_with_weight"],
                 "score": c["similarity"],
                 "title": c["docnm_kwd"],
-                "metadata": {}
+                "metadata": doc.meta_fields
             })
 
         return jsonify({"records": records})

api/apps/sdk/doc.py

@@ -13,37 +13,35 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import pathlib
 import datetime
-
-from api.db.services.dialog_service import keyword_extraction, label_question
-from rag.app.qa import rmPrefix, beAdoc
-from rag.nlp import rag_tokenizer
-from api.db import LLMType, ParserType
-from api.db.services.llm_service import TenantLLMService, LLMBundle
-from api import settings
-import xxhash
+import logging
+import pathlib
 import re
-from api.utils.api_utils import token_required
-from api.db.db_models import Task
-from api.db.services.task_service import TaskService, queue_tasks
-from api.utils.api_utils import server_error_response
-from api.utils.api_utils import get_result, get_error_data_result
 from io import BytesIO
+
+import xxhash
 from flask import request, send_file
-from api.db import FileSource, TaskStatus, FileType
-from api.db.db_models import File
+from peewee import OperationalError
+from pydantic import BaseModel, Field, validator
+
+from api import settings
+from api.constants import FILE_NAME_LEN_LIMIT
+from api.db import FileSource, FileType, LLMType, ParserType, TaskStatus
+from api.db.db_models import File, Task
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.utils.api_utils import construct_json_result, get_parser_config
-from rag.nlp import search
+from api.db.services.llm_service import LLMBundle, TenantLLMService
+from api.db.services.task_service import TaskService, queue_tasks
+from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_parser_config, get_result, server_error_response, token_required
+from rag.app.qa import beAdoc, rmPrefix
+from rag.app.tag import label_question
+from rag.nlp import rag_tokenizer, search
+from rag.prompts import keyword_extraction
 from rag.utils import rmSpace
 from rag.utils.storage_factory import STORAGE_IMPL
 
-from pydantic import BaseModel, Field, validator
-
 MAXIMUM_OF_UPLOADING_FILES = 256
 
 
@@ -59,13 +57,14 @@ class Chunk(BaseModel):
     available: bool = True
     positions: list[list[int]] = Field(default_factory=list)
 
-    @validator('positions')
+    @validator("positions")
     def validate_positions(cls, value):
         for sublist in value:
             if len(sublist) != 5:
                 raise ValueError("Each sublist in positions must have a length of 5")
         return value
 
+
 @manager.route("/datasets/<dataset_id>/documents", methods=["POST"])  # noqa: F821
 @token_required
 def upload(dataset_id, tenant_id):
@@ -126,16 +125,14 @@ def upload(dataset_id, tenant_id):
                     description: Processing status.
     """
     if "file" not in request.files:
-        return get_error_data_result(
-            message="No file part!", code=settings.RetCode.ARGUMENT_ERROR
-        )
+        return get_error_data_result(message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
     file_objs = request.files.getlist("file")
     for file_obj in file_objs:
         if file_obj.filename == "":
-            return get_result(
-                message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR
-            )
-    '''
+            return get_result(message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
+        if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+            return get_result(message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+    """
     # total size
     total_size = 0
     for file_obj in file_objs:
@@ -148,7 +145,7 @@ def upload(dataset_id, tenant_id):
             message=f"Total file size exceeds 10MB limit! ({total_size / (1024 * 1024):.2f} MB)",
             code=settings.RetCode.ARGUMENT_ERROR,
         )
-    '''
+    """
     e, kb = KnowledgebaseService.get_by_id(dataset_id)
     if not e:
         raise LookupError(f"Can't find the dataset with ID {dataset_id}!")
@@ -216,6 +213,9 @@ def update_doc(tenant_id, dataset_id, document_id):
             chunk_method:
               type: string
               description: Chunking method.
+            enabled:
+              type: boolean
+              description: Document status.
     responses:
       200:
         description: Document updated successfully.
@@ -225,6 +225,9 @@ def update_doc(tenant_id, dataset_id, document_id):
     req = request.json
     if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
         return get_error_data_result(message="You don't own the dataset.")
+    e, kb = KnowledgebaseService.get_by_id(dataset_id)
+    if not e:
+        return get_error_data_result(message="Can't find this knowledgebase!")
     doc = DocumentService.query(kb_id=dataset_id, id=document_id)
     if not doc:
         return get_error_data_result(message="The dataset doesn't own the document.")
@@ -239,20 +242,25 @@ def update_doc(tenant_id, dataset_id, document_id):
         if req["progress"] != doc.progress:
             return get_error_data_result(message="Can't change `progress`.")
 
+    if "meta_fields" in req:
+        if not isinstance(req["meta_fields"], dict):
+            return get_error_data_result(message="meta_fields must be a dictionary")
+        DocumentService.update_meta_fields(document_id, req["meta_fields"])
+
     if "name" in req and req["name"] != doc.name:
-        if (
-                pathlib.Path(req["name"].lower()).suffix
-                != pathlib.Path(doc.name.lower()).suffix
-        ):
+        if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+            return get_result(
+                message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.",
+                code=settings.RetCode.ARGUMENT_ERROR,
+            )
+        if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
             return get_result(
                 message="The extension of file can't be changed",
                 code=settings.RetCode.ARGUMENT_ERROR,
             )
         for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
             if d.name == req["name"]:
-                return get_error_data_result(
-                    message="Duplicated document name in the same dataset."
-                )
+                return get_error_data_result(message="Duplicated document name in the same dataset.")
         if not DocumentService.update_by_id(document_id, {"name": req["name"]}):
             return get_error_data_result(message="Database error (Document rename)!")
 
@@ -260,49 +268,32 @@ def update_doc(tenant_id, dataset_id, document_id):
         if informs:
             e, file = FileService.get_by_id(informs[0].file_id)
             FileService.update_by_id(file.id, {"name": req["name"]})
+
     if "parser_config" in req:
         DocumentService.update_parser_config(doc.id, req["parser_config"])
     if "chunk_method" in req:
-        valid_chunk_method = {
-            "naive",
-            "manual",
-            "qa",
-            "table",
-            "paper",
-            "book",
-            "laws",
-            "presentation",
-            "picture",
-            "one",
-            "knowledge_graph",
-            "email",
-            "tag"
-        }
+        valid_chunk_method = {"naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one", "knowledge_graph", "email", "tag"}
         if req.get("chunk_method") not in valid_chunk_method:
-            return get_error_data_result(
-                f"`chunk_method` {req['chunk_method']} doesn't exist"
-            )
-        if doc.parser_id.lower() == req["chunk_method"].lower():
-            return get_result()
+            return get_error_data_result(f"`chunk_method` {req['chunk_method']} doesn't exist")
 
         if doc.type == FileType.VISUAL or re.search(r"\.(ppt|pptx|pages)$", doc.name):
             return get_error_data_result(message="Not supported yet!")
 
-        e = DocumentService.update_by_id(
-            doc.id,
-            {
-                "parser_id": req["chunk_method"],
-                "progress": 0,
-                "progress_msg": "",
-                "run": TaskStatus.UNSTART.value,
-            },
-        )
-        if not e:
-            return get_error_data_result(message="Document not found!")
-        req["parser_config"] = get_parser_config(
-            req["chunk_method"], req.get("parser_config")
-        )
-        DocumentService.update_parser_config(doc.id, req["parser_config"])
+        if doc.parser_id.lower() != req["chunk_method"].lower():
+            e = DocumentService.update_by_id(
+                doc.id,
+                {
+                    "parser_id": req["chunk_method"],
+                    "progress": 0,
+                    "progress_msg": "",
+                    "run": TaskStatus.UNSTART.value,
+                },
+            )
+            if not e:
+                return get_error_data_result(message="Document not found!")
+        if not req.get("parser_config"):
+            req["parser_config"] = get_parser_config(req["chunk_method"], req.get("parser_config"))
+            DocumentService.update_parser_config(doc.id, req["parser_config"])
         if doc.token_num > 0:
             e = DocumentService.increment_chunk_num(
                 doc.id,
@@ -315,7 +306,49 @@ def update_doc(tenant_id, dataset_id, document_id):
                 return get_error_data_result(message="Document not found!")
             settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), dataset_id)
 
-    return get_result()
+    if "enabled" in req:
+        status = int(req["enabled"])
+        if doc.status != req["enabled"]:
+            try:
+                if not DocumentService.update_by_id(doc.id, {"status": str(status)}):
+                    return get_error_data_result(message="Database error (Document update)!")
+
+                settings.docStoreConn.update({"doc_id": doc.id}, {"available_int": status}, search.index_name(kb.tenant_id), doc.kb_id)
+                return get_result(data=True)
+            except Exception as e:
+                return server_error_response(e)
+
+    try:
+        ok, doc = DocumentService.get_by_id(doc.id)
+        if not ok:
+            return get_error_data_result(message="Dataset created failed")
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
+
+    key_mapping = {
+        "chunk_num": "chunk_count",
+        "kb_id": "dataset_id",
+        "token_num": "token_count",
+        "parser_id": "chunk_method",
+    }
+    run_mapping = {
+        "0": "UNSTART",
+        "1": "RUNNING",
+        "2": "CANCEL",
+        "3": "DONE",
+        "4": "FAIL",
+    }
+    renamed_doc = {}
+    for key, value in doc.to_dict().items():
+        if key == "run":
+            renamed_doc["run"] = run_mapping.get(str(value))
+        new_key = key_mapping.get(key, key)
+        renamed_doc[new_key] = value
+        if key == "run":
+            renamed_doc["run"] = run_mapping.get(value)
+
+    return get_result(data=renamed_doc)
 
 
 @manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["GET"])  # noqa: F821
@@ -356,22 +389,18 @@ def download(tenant_id, dataset_id, document_id):
         schema:
           type: object
     """
+    if not document_id:
+        return get_error_data_result(message="Specify document_id please.")
     if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
         return get_error_data_result(message=f"You do not own the dataset {dataset_id}.")
     doc = DocumentService.query(kb_id=dataset_id, id=document_id)
     if not doc:
-        return get_error_data_result(
-            message=f"The dataset not own the document {document_id}."
-        )
+        return get_error_data_result(message=f"The dataset not own the document {document_id}.")
     # The process of downloading
-    doc_id, doc_location = File2DocumentService.get_storage_address(
-        doc_id=document_id
-    )  # minio address
+    doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id)  # minio address
     file_stream = STORAGE_IMPL.get(doc_id, doc_location)
     if not file_stream:
-        return construct_json_result(
-            message="This file is empty.", code=settings.RetCode.DATA_ERROR
-        )
+        return construct_json_result(message="This file is empty.", code=settings.RetCode.DATA_ERROR)
     file = BytesIO(file_stream)
     # Use send_file with a proper filename and MIME type
     return send_file(
@@ -472,10 +501,12 @@ def list_docs(dataset_id, tenant_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}. ")
     id = request.args.get("id")
     name = request.args.get("name")
-    if not DocumentService.query(id=id, kb_id=dataset_id):
+
+    if id and not DocumentService.query(id=id, kb_id=dataset_id):
         return get_error_data_result(message=f"You don't own the document {id}.")
-    if not DocumentService.query(name=name, kb_id=dataset_id):
+    if name and not DocumentService.query(name=name, kb_id=dataset_id):
         return get_error_data_result(message=f"You don't own the document {name}.")
+
     page = int(request.args.get("page", 1))
     keywords = request.args.get("keywords", "")
     page_size = int(request.args.get("page_size", 30))
@@ -484,9 +515,7 @@ def list_docs(dataset_id, tenant_id):
         desc = False
     else:
         desc = True
-    docs, tol = DocumentService.get_list(
-        dataset_id, page, page_size, orderby, desc, keywords, id, name
-    )
+    docs, tol = DocumentService.get_list(dataset_id, page, page_size, orderby, desc, keywords, id, name)
 
     # rename key's name
     renamed_doc_list = []
@@ -569,15 +598,22 @@ def delete(tenant_id, dataset_id):
             doc_list.append(doc.id)
     else:
         doc_list = doc_ids
+
+    unique_doc_ids, duplicate_messages = check_duplicate_ids(doc_list, "document")
+    doc_list = unique_doc_ids
+
     root_folder = FileService.get_root_folder(tenant_id)
     pf_id = root_folder["id"]
     FileService.init_knowledgebase_docs(pf_id, tenant_id)
     errors = ""
+    not_found = []
+    success_count = 0
     for doc_id in doc_list:
         try:
             e, doc = DocumentService.get_by_id(doc_id)
             if not e:
-                return get_error_data_result(message="Document not found!")
+                not_found.append(doc_id)
+                continue
             tenant_id = DocumentService.get_tenant_id(doc_id)
             if not tenant_id:
                 return get_error_data_result(message="Tenant not found!")
@@ -585,9 +621,7 @@ def delete(tenant_id, dataset_id):
             b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
 
             if not DocumentService.remove_document(doc, tenant_id):
-                return get_error_data_result(
-                    message="Database error (Document removal)!"
-                )
+                return get_error_data_result(message="Database error (Document removal)!")
 
             f2d = File2DocumentService.get_by_document_id(doc_id)
             FileService.filter_delete(
@@ -599,12 +633,25 @@ def delete(tenant_id, dataset_id):
             File2DocumentService.delete_by_document_id(doc_id)
 
             STORAGE_IMPL.rm(b, n)
+            success_count += 1
         except Exception as e:
             errors += str(e)
 
+    if not_found:
+        return get_result(message=f"Documents not found: {not_found}", code=settings.RetCode.DATA_ERROR)
+
     if errors:
         return get_result(message=errors, code=settings.RetCode.SERVER_ERROR)
 
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(
+                message=f"Partially deleted {success_count} datasets with {len(duplicate_messages)} errors",
+                data={"success_count": success_count, "errors": duplicate_messages},
+            )
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+
     return get_result()
 
 
@@ -652,18 +699,22 @@ def parse(tenant_id, dataset_id):
     req = request.json
     if not req.get("document_ids"):
         return get_error_data_result("`document_ids` is required")
-    for id in req["document_ids"]:
+    doc_list = req.get("document_ids")
+    unique_doc_ids, duplicate_messages = check_duplicate_ids(doc_list, "document")
+    doc_list = unique_doc_ids
+
+    not_found = []
+    success_count = 0
+    for id in doc_list:
         doc = DocumentService.query(id=id, kb_id=dataset_id)
+        if not doc:
+            not_found.append(id)
+            continue
         if not doc:
             return get_error_data_result(message=f"You don't own the document {id}.")
-        if doc[0].progress != 0.0:
-            return get_error_data_result(
-                "Can't stop parsing document with progress at 0 or 100"
-            )
-        info = {"run": "1", "progress": 0}
-        info["progress_msg"] = ""
-        info["chunk_num"] = 0
-        info["token_num"] = 0
+        if 0.0 < doc[0].progress < 1.0:
+            return get_error_data_result("Can't parse document that is currently being processed")
+        info = {"run": "1", "progress": 0, "progress_msg": "", "chunk_num": 0, "token_num": 0}
         DocumentService.update_by_id(id, info)
         settings.docStoreConn.delete({"doc_id": id}, search.index_name(tenant_id), dataset_id)
         TaskService.filter_delete([Task.doc_id == id])
@@ -671,7 +722,19 @@ def parse(tenant_id, dataset_id):
         doc = doc.to_dict()
         doc["tenant_id"] = tenant_id
         bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
-        queue_tasks(doc, bucket, name)
+        queue_tasks(doc, bucket, name, 0)
+        success_count += 1
+    if not_found:
+        return get_result(message=f"Documents not found: {not_found}", code=settings.RetCode.DATA_ERROR)
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(
+                message=f"Partially parsed {success_count} documents with {len(duplicate_messages)} errors",
+                data={"success_count": success_count, "errors": duplicate_messages},
+            )
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+
     return get_result()
 
 
@@ -717,19 +780,32 @@ def stop_parsing(tenant_id, dataset_id):
     if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
     req = request.json
+
     if not req.get("document_ids"):
         return get_error_data_result("`document_ids` is required")
-    for id in req["document_ids"]:
+    doc_list = req.get("document_ids")
+    unique_doc_ids, duplicate_messages = check_duplicate_ids(doc_list, "document")
+    doc_list = unique_doc_ids
+
+    success_count = 0
+    for id in doc_list:
         doc = DocumentService.query(id=id, kb_id=dataset_id)
         if not doc:
             return get_error_data_result(message=f"You don't own the document {id}.")
         if int(doc[0].progress) == 1 or doc[0].progress == 0:
-            return get_error_data_result(
-                "Can't stop parsing document with progress at 0 or 1"
-            )
+            return get_error_data_result("Can't stop parsing document with progress at 0 or 1")
         info = {"run": "2", "progress": 0, "chunk_num": 0}
         DocumentService.update_by_id(id, info)
-        settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), dataset_id)
+        settings.docStoreConn.delete({"doc_id": doc[0].id}, search.index_name(tenant_id), dataset_id)
+        success_count += 1
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(
+                message=f"Partially stopped {success_count} documents with {len(duplicate_messages)} errors",
+                data={"success_count": success_count, "errors": duplicate_messages},
+            )
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
     return get_result()
 
 
@@ -766,6 +842,12 @@ def list_chunks(tenant_id, dataset_id, document_id):
         required: false
         default: 30
         description: Number of items per page.
+      - in: query
+        name: id
+        type: string
+        required: false
+        default: ""
+        description: Chunk Id.
       - in: header
         name: Authorization
         type: string
@@ -810,9 +892,7 @@ def list_chunks(tenant_id, dataset_id, document_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
     doc = DocumentService.query(id=document_id, kb_id=dataset_id)
     if not doc:
-        return get_error_data_result(
-            message=f"You don't own the document {document_id}."
-        )
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
     doc = doc[0]
     req = request.args
     doc_id = document_id
@@ -850,6 +930,8 @@ def list_chunks(tenant_id, dataset_id, document_id):
     res = {"total": 0, "chunks": [], "doc": renamed_doc}
     if req.get("id"):
         chunk = settings.docStoreConn.get(req.get("id"), search.index_name(tenant_id), [dataset_id])
+        if not chunk:
+            return get_result(message=f"Chunk not found: {dataset_id}/{req.get('id')}", code=settings.RetCode.NOT_FOUND)
         k = []
         for n in chunk.keys():
             if re.search(r"(_vec$|_sm_|_tks|_ltks)", n):
@@ -858,45 +940,40 @@ def list_chunks(tenant_id, dataset_id, document_id):
             del chunk[n]
         if not chunk:
             return get_error_data_result(f"Chunk `{req.get('id')}` not found.")
-        res['total'] = 1
+        res["total"] = 1
         final_chunk = {
-            "id":chunk.get("id",chunk.get("chunk_id")),
-            "content":chunk["content_with_weight"],
-            "document_id":chunk.get("doc_id",chunk.get("document_id")),
-            "docnm_kwd":chunk["docnm_kwd"],
-            "important_keywords":chunk.get("important_kwd",[]),
-            "questions":chunk.get("question_kwd",[]),
-            "dataset_id":chunk.get("kb_id",chunk.get("dataset_id")),
-            "image_id":chunk["img_id"],
-            "available":bool(chunk.get("available_int",1)),
-            "positions":chunk.get("position_int",[]),
+            "id": chunk.get("id", chunk.get("chunk_id")),
+            "content": chunk["content_with_weight"],
+            "document_id": chunk.get("doc_id", chunk.get("document_id")),
+            "docnm_kwd": chunk["docnm_kwd"],
+            "important_keywords": chunk.get("important_kwd", []),
+            "questions": chunk.get("question_kwd", []),
+            "dataset_id": chunk.get("kb_id", chunk.get("dataset_id")),
+            "image_id": chunk.get("img_id", ""),
+            "available": bool(chunk.get("available_int", 1)),
+            "positions": chunk.get("position_int", []),
         }
         res["chunks"].append(final_chunk)
         _ = Chunk(**final_chunk)
 
     elif settings.docStoreConn.indexExist(search.index_name(tenant_id), dataset_id):
-        sres = settings.retrievaler.search(query, search.index_name(tenant_id), [dataset_id], emb_mdl=None,
-                                           highlight=True)
+        sres = settings.retrievaler.search(query, search.index_name(tenant_id), [dataset_id], emb_mdl=None, highlight=True)
         res["total"] = sres.total
         for id in sres.ids:
             d = {
                 "id": id,
-                "content": (
-                    rmSpace(sres.highlight[id])
-                    if question and id in sres.highlight
-                    else sres.field[id].get("content_with_weight", "")
-                ),
+                "content": (rmSpace(sres.highlight[id]) if question and id in sres.highlight else sres.field[id].get("content_with_weight", "")),
                 "document_id": sres.field[id]["doc_id"],
                 "docnm_kwd": sres.field[id]["docnm_kwd"],
                 "important_keywords": sres.field[id].get("important_kwd", []),
                 "questions": sres.field[id].get("question_kwd", []),
                 "dataset_id": sres.field[id].get("kb_id", sres.field[id].get("dataset_id")),
                 "image_id": sres.field[id].get("img_id", ""),
-                "available": bool(sres.field[id].get("available_int", 1)),
-                "positions": sres.field[id].get("position_int",[]),
+                "available": bool(int(sres.field[id].get("available_int", "1"))),
+                "positions": sres.field[id].get("position_int", []),
             }
             res["chunks"].append(d)
-            _ = Chunk(**d) # validate the chunk
+            _ = Chunk(**d)  # validate the chunk
     return get_result(data=res)
 
 
@@ -972,23 +1049,17 @@ def add_chunk(tenant_id, dataset_id, document_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
     doc = DocumentService.query(id=document_id, kb_id=dataset_id)
     if not doc:
-        return get_error_data_result(
-            message=f"You don't own the document {document_id}."
-        )
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
     doc = doc[0]
     req = request.json
-    if not req.get("content"):
+    if not str(req.get("content", "")).strip():
         return get_error_data_result(message="`content` is required")
     if "important_keywords" in req:
         if not isinstance(req["important_keywords"], list):
-            return get_error_data_result(
-                "`important_keywords` is required to be a list"
-            )
+            return get_error_data_result("`important_keywords` is required to be a list")
     if "questions" in req:
         if not isinstance(req["questions"], list):
-            return get_error_data_result(
-                "`questions` is required to be a list"
-            )
+            return get_error_data_result("`questions` is required to be a list")
     chunk_id = xxhash.xxh64((req["content"] + document_id).encode("utf-8")).hexdigest()
     d = {
         "id": chunk_id,
@@ -997,22 +1068,16 @@ def add_chunk(tenant_id, dataset_id, document_id):
     }
     d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
     d["important_kwd"] = req.get("important_keywords", [])
-    d["important_tks"] = rag_tokenizer.tokenize(
-        " ".join(req.get("important_keywords", []))
-    )
-    d["question_kwd"] = req.get("questions", [])
-    d["question_tks"] = rag_tokenizer.tokenize(
-        "\n".join(req.get("questions", []))
-    )
+    d["important_tks"] = rag_tokenizer.tokenize(" ".join(req.get("important_keywords", [])))
+    d["question_kwd"] = [str(q).strip() for q in req.get("questions", []) if str(q).strip()]
+    d["question_tks"] = rag_tokenizer.tokenize("\n".join(req.get("questions", [])))
     d["create_time"] = str(datetime.datetime.now()).replace("T", " ")[:19]
     d["create_timestamp_flt"] = datetime.datetime.now().timestamp()
     d["kb_id"] = dataset_id
     d["docnm_kwd"] = doc.name
     d["doc_id"] = document_id
     embd_id = DocumentService.get_embd_id(document_id)
-    embd_mdl = TenantLLMService.model_instance(
-        tenant_id, LLMType.EMBEDDING.value, embd_id
-    )
+    embd_mdl = TenantLLMService.model_instance(tenant_id, LLMType.EMBEDDING.value, embd_id)
     v, c = embd_mdl.encode([doc.name, req["content"] if not d["question_kwd"] else "\n".join(d["question_kwd"])])
     v = 0.1 * v[0] + 0.9 * v[1]
     d["q_%d_vec" % len(v)] = v.tolist()
@@ -1089,15 +1154,26 @@ def rm_chunk(tenant_id, dataset_id, document_id):
     """
     if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    docs = DocumentService.get_by_ids([document_id])
+    if not docs:
+        raise LookupError(f"Can't find the document with ID {document_id}!")
     req = request.json
     condition = {"doc_id": document_id}
     if "chunk_ids" in req:
-        condition["id"] = req["chunk_ids"]
+        unique_chunk_ids, duplicate_messages = check_duplicate_ids(req["chunk_ids"], "chunk")
+        condition["id"] = unique_chunk_ids
     chunk_number = settings.docStoreConn.delete(condition, search.index_name(tenant_id), dataset_id)
     if chunk_number != 0:
         DocumentService.decrement_chunk_num(document_id, dataset_id, 1, chunk_number, 0)
-    if "chunk_ids" in req and chunk_number != len(req["chunk_ids"]):
-        return get_error_data_result(message=f"rm_chunk deleted chunks {chunk_number}, expect {len(req['chunk_ids'])}")
+    if "chunk_ids" in req and chunk_number != len(unique_chunk_ids):
+        if len(unique_chunk_ids) == 0:
+            return get_result(message=f"deleted {chunk_number} chunks")
+        return get_error_data_result(message=f"rm_chunk deleted chunks {chunk_number}, expect {len(unique_chunk_ids)}")
+    if duplicate_messages:
+        return get_result(
+            message=f"Partially deleted {chunk_number} chunks with {len(duplicate_messages)} errors",
+            data={"success_count": chunk_number, "errors": duplicate_messages},
+        )
     return get_result(message=f"deleted {chunk_number} chunks")
 
 
@@ -1165,9 +1241,7 @@ def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
     doc = DocumentService.query(id=document_id, kb_id=dataset_id)
     if not doc:
-        return get_error_data_result(
-            message=f"You don't own the document {document_id}."
-        )
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
     doc = doc[0]
     req = request.json
     if "content" in req:
@@ -1185,24 +1259,18 @@ def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
     if "questions" in req:
         if not isinstance(req["questions"], list):
             return get_error_data_result("`questions` should be a list")
-        d["question_kwd"] = req.get("questions")
+        d["question_kwd"] = [str(q).strip() for q in req.get("questions", []) if str(q).strip()]
         d["question_tks"] = rag_tokenizer.tokenize("\n".join(req["questions"]))
     if "available" in req:
         d["available_int"] = int(req["available"])
     embd_id = DocumentService.get_embd_id(document_id)
-    embd_mdl = TenantLLMService.model_instance(
-        tenant_id, LLMType.EMBEDDING.value, embd_id
-    )
+    embd_mdl = TenantLLMService.model_instance(tenant_id, LLMType.EMBEDDING.value, embd_id)
     if doc.parser_id == ParserType.QA:
         arr = [t for t in re.split(r"[\n\t]", d["content_with_weight"]) if len(t) > 1]
         if len(arr) != 2:
-            return get_error_data_result(
-                message="Q&A must be separated by TAB/ENTER key."
-            )
+            return get_error_data_result(message="Q&A must be separated by TAB/ENTER key.")
         q, a = rmPrefix(arr[0]), rmPrefix(arr[1])
-        d = beAdoc(
-            d, arr[0], arr[1], not any([rag_tokenizer.is_chinese(t) for t in q + a])
-        )
+        d = beAdoc(d, arr[0], arr[1], not any([rag_tokenizer.is_chinese(t) for t in q + a]))
 
     v, c = embd_mdl.encode([doc.name, d["content_with_weight"] if not d.get("question_kwd") else "\n".join(d["question_kwd"])])
     v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
@@ -1301,7 +1369,7 @@ def retrieval_test(tenant_id):
         if not KnowledgebaseService.accessible(kb_id=id, user_id=tenant_id):
             return get_error_data_result(f"You don't own the dataset {id}.")
     kbs = KnowledgebaseService.get_by_ids(kb_ids)
-    embd_nms = list(set([kb.embd_id for kb in kbs]))
+    embd_nms = list(set([TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]))  # remove vendor suffix for comparison
     if len(embd_nms) != 1:
         return get_result(
             message='Datasets use different embedding models."',
@@ -1319,9 +1387,7 @@ def retrieval_test(tenant_id):
     doc_ids_list = KnowledgebaseService.list_documents_by_ids(kb_ids)
     for doc_id in doc_ids:
         if doc_id not in doc_ids_list:
-            return get_error_data_result(
-                f"The datasets don't own the document {doc_id}"
-            )
+            return get_error_data_result(f"The datasets don't own the document {doc_id}")
     similarity_threshold = float(req.get("similarity_threshold", 0.2))
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
     top = int(req.get("top_k", 1024))
@@ -1330,6 +1396,7 @@ def retrieval_test(tenant_id):
     else:
         highlight = True
     try:
+        tenant_ids = list(set([kb.tenant_id for kb in kbs]))
         e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
         if not e:
             return get_error_data_result(message="Dataset not found!")
@@ -1346,7 +1413,7 @@ def retrieval_test(tenant_id):
         ranks = settings.retrievaler.retrieval(
             question,
             embd_mdl,
-            kb.tenant_id,
+            tenant_ids,
             kb_ids,
             page,
             size,
@@ -1356,14 +1423,10 @@ def retrieval_test(tenant_id):
             doc_ids,
             rerank_mdl=rerank_mdl,
             highlight=highlight,
-            rank_feature=label_question(question, kbs)
+            rank_feature=label_question(question, kbs),
         )
         if use_kg:
-            ck = settings.kg_retrievaler.retrieval(question,
-                                                   [k.tenant_id for k in kbs],
-                                                   kb_ids,
-                                                   embd_mdl,
-                                                   LLMBundle(kb.tenant_id, LLMType.CHAT))
+            ck = settings.kg_retrievaler.retrieval(question, [k.tenant_id for k in kbs], kb_ids, embd_mdl, LLMBundle(kb.tenant_id, LLMType.CHAT))
             if ck["content_with_weight"]:
                 ranks["chunks"].insert(0, ck)
 
@@ -1380,7 +1443,7 @@ def retrieval_test(tenant_id):
                 "important_kwd": "important_keywords",
                 "question_kwd": "questions",
                 "docnm_kwd": "document_keyword",
-                "kb_id":"dataset_id"
+                "kb_id": "dataset_id",
             }
             rename_chunk = {}
             for key, value in chunk.items():

api/apps/sdk/session.py

@@ -13,30 +13,30 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import re
 import json
-from api.db import LLMType
-from flask import request, Response
+import re
+import time
 
+import tiktoken
+from flask import Response, jsonify, request
 from api.db.services.conversation_service import ConversationService, iframe_completion
 from api.db.services.conversation_service import completion as rag_completion
-from api.db.services.canvas_service import completion as agent_completion
-from api.db.services.dialog_service import ask
+from api.db.services.canvas_service import completion as agent_completion, completionOpenAI
 from agent.canvas import Canvas
-from api.db import StatusEnum
+from api.db import LLMType, StatusEnum
 from api.db.db_models import APIToken
 from api.db.services.api_service import API4ConversationService
 from api.db.services.canvas_service import UserCanvasService
-from api.db.services.dialog_service import DialogService
+from api.db.services.dialog_service import DialogService, ask, chat
+from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.utils import get_uuid
-from api.utils.api_utils import get_error_data_result
-from api.utils.api_utils import get_result, token_required
+from api.utils.api_utils import get_result, token_required, get_data_openai, get_error_data_result, validate_request, check_duplicate_ids
 from api.db.services.llm_service import LLMBundle
 
 
 
-@manager.route('/chats/<chat_id>/sessions', methods=['POST'])  # noqa: F821
+@manager.route("/chats/<chat_id>/sessions", methods=["POST"])  # noqa: F821
 @token_required
 def create(tenant_id, chat_id):
     req = request.json
@@ -49,7 +49,7 @@ def create(tenant_id, chat_id):
         "dialog_id": req["dialog_id"],
         "name": req.get("name", "New session"),
         "message": [{"role": "assistant", "content": dia[0].prompt_config.get("prologue")}],
-        "user_id": req.get("user_id", "")
+        "user_id": req.get("user_id", ""),
     }
     if not conv.get("name"):
         return get_error_data_result(message="`name` can not be empty.")
@@ -58,23 +58,25 @@ def create(tenant_id, chat_id):
     if not e:
         return get_error_data_result(message="Fail to create a session!")
     conv = conv.to_dict()
-    conv['messages'] = conv.pop("message")
+    conv["messages"] = conv.pop("message")
     conv["chat_id"] = conv.pop("dialog_id")
     del conv["reference"]
     return get_result(data=conv)
 
 
-@manager.route('/agents/<agent_id>/sessions', methods=['POST'])  # noqa: F821
+@manager.route("/agents/<agent_id>/sessions", methods=["POST"])  # noqa: F821
 @token_required
 def create_agent_session(tenant_id, agent_id):
     req = request.json
+    if not request.is_json:
+        req = request.form
+    files = request.files
+    user_id = request.args.get("user_id", "")
     e, cvs = UserCanvasService.get_by_id(agent_id)
     if not e:
         return get_error_data_result("Agent not found.")
-
     if not UserCanvasService.query(user_id=tenant_id, id=agent_id):
         return get_error_data_result("You cannot access the agent.")
-
     if not isinstance(cvs.dsl, str):
         cvs.dsl = json.dumps(cvs.dsl, ensure_ascii=False)
 
@@ -84,33 +86,45 @@ def create_agent_session(tenant_id, agent_id):
     if query:
         for ele in query:
             if not ele["optional"]:
-                if not req.get(ele["key"]):
-                    return get_error_data_result(f"`{ele['key']}` is required")
-                ele["value"] = req[ele["key"]]
-            if ele["optional"]:
-                if req.get(ele["key"]):
-                    ele["value"] = req[ele['key']]
+                if ele["type"] == "file":
+                    if files is None or not files.get(ele["key"]):
+                        return get_error_data_result(f"`{ele['key']}` with type `{ele['type']}` is required")
+                    upload_file = files.get(ele["key"])
+                    file_content = FileService.parse_docs([upload_file], user_id)
+                    file_name = upload_file.filename
+                    ele["value"] = file_name + "\n" + file_content
                 else:
-                    if "value" in ele:
-                        ele.pop("value")
-    else:
-        for ans in canvas.run(stream=False):
-            pass
+                    if req is None or not req.get(ele["key"]):
+                        return get_error_data_result(f"`{ele['key']}` with type `{ele['type']}` is required")
+                    ele["value"] = req[ele["key"]]
+            else:
+                if ele["type"] == "file":
+                    if files is not None and files.get(ele["key"]):
+                        upload_file = files.get(ele["key"])
+                        file_content = FileService.parse_docs([upload_file], user_id)
+                        file_name = upload_file.filename
+                        ele["value"] = file_name + "\n" + file_content
+                    else:
+                        if "value" in ele:
+                            ele.pop("value")
+                else:
+                    if req is not None and req.get(ele["key"]):
+                        ele["value"] = req[ele["key"]]
+                    else:
+                        if "value" in ele:
+                            ele.pop("value")
+
+    for ans in canvas.run(stream=False):
+        pass
+
     cvs.dsl = json.loads(str(canvas))
-    conv = {
-        "id": get_uuid(),
-        "dialog_id": cvs.id,
-        "user_id": req.get("user_id", "") if isinstance(req, dict) else "",
-        "message": [{"role": "assistant", "content": canvas.get_prologue()}],
-        "source": "agent",
-        "dsl": cvs.dsl
-    }
+    conv = {"id": get_uuid(), "dialog_id": cvs.id, "user_id": user_id, "message": [{"role": "assistant", "content": canvas.get_prologue()}], "source": "agent", "dsl": cvs.dsl}
     API4ConversationService.save(**conv)
     conv["agent_id"] = conv.pop("dialog_id")
     return get_result(data=conv)
 
 
-@manager.route('/chats/<chat_id>/sessions/<session_id>', methods=['PUT'])  # noqa: F821
+@manager.route("/chats/<chat_id>/sessions/<session_id>", methods=["PUT"])  # noqa: F821
 @token_required
 def update(tenant_id, chat_id, session_id):
     req = request.json
@@ -132,12 +146,14 @@ def update(tenant_id, chat_id, session_id):
     return get_result()
 
 
-@manager.route('/chats/<chat_id>/completions', methods=['POST'])  # noqa: F821
+@manager.route("/chats/<chat_id>/completions", methods=["POST"])  # noqa: F821
 @token_required
 def chat_completion(tenant_id, chat_id):
     req = request.json
-    if not req or not req.get("session_id"):
+    if not req:
         req = {"question": ""}
+    if not req.get("session_id"):
+        req["question"] = ""
     if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
         return get_error_data_result(f"You don't own the chat {chat_id}")
     if req.get("session_id"):
@@ -159,7 +175,227 @@ def chat_completion(tenant_id, chat_id):
         return get_result(data=answer)
 
 
-@manager.route('/agents/<agent_id>/completions', methods=['POST'])  # noqa: F821
+@manager.route("/chats_openai/<chat_id>/chat/completions", methods=["POST"])  # noqa: F821
+@validate_request("model", "messages")  # noqa: F821
+@token_required
+def chat_completion_openai_like(tenant_id, chat_id):
+    """
+    OpenAI-like chat completion API that simulates the behavior of OpenAI's completions endpoint.
+    
+    This function allows users to interact with a model and receive responses based on a series of historical messages.
+    If `stream` is set to True (by default), the response will be streamed in chunks, mimicking the OpenAI-style API.
+    Set `stream` to False explicitly, the response will be returned in a single complete answer.
+    Example usage:
+
+    curl -X POST https://ragflow_address.com/api/v1/chats_openai/<chat_id>/chat/completions \
+        -H "Content-Type: application/json" \
+        -H "Authorization: Bearer $RAGFLOW_API_KEY" \
+        -d '{
+            "model": "model",
+            "messages": [{"role": "user", "content": "Say this is a test!"}],
+            "stream": true
+        }'
+
+    Alternatively, you can use Python's `OpenAI` client:
+
+    from openai import OpenAI
+
+    model = "model"
+    client = OpenAI(api_key="ragflow-api-key", base_url=f"http://ragflow_address/api/v1/chats_openai/<chat_id>")
+    
+    completion = client.chat.completions.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Who are you?"},
+            {"role": "assistant", "content": "I am an AI assistant named..."},
+            {"role": "user", "content": "Can you tell me how to install neovim"},
+        ],
+        stream=True
+    )
+    
+    stream = True
+    if stream:
+        for chunk in completion:
+            print(chunk)
+    else:
+        print(completion.choices[0].message.content)
+    """
+    req = request.json
+
+    messages = req.get("messages", [])
+    # To prevent empty [] input
+    if len(messages) < 1:
+        return get_error_data_result("You have to provide messages.")
+    if messages[-1]["role"] != "user":
+        return get_error_data_result("The last content of this conversation is not from user.")
+
+    prompt = messages[-1]["content"]
+    # Treat context tokens as reasoning tokens
+    context_token_used = sum(len(message["content"]) for message in messages)
+
+    dia = DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value)
+    if not dia:
+        return get_error_data_result(f"You don't own the chat {chat_id}")
+    dia = dia[0]
+
+    # Filter system and non-sense assistant messages
+    msg = []
+    for m in messages:
+        if m["role"] == "system":
+            continue
+        if m["role"] == "assistant" and not msg:
+            continue
+        msg.append(m)
+
+    # tools = get_tools()
+    # toolcall_session = SimpleFunctionCallServer()
+    tools = None
+    toolcall_session = None
+
+    if req.get("stream", True):
+        # The value for the usage field on all chunks except for the last one will be null.
+        # The usage field on the last chunk contains token usage statistics for the entire request.
+        # The choices field on the last chunk will always be an empty array [].
+        def streamed_response_generator(chat_id, dia, msg):
+            token_used = 0
+            answer_cache = ""
+            reasoning_cache = ""
+            response = {
+                "id": f"chatcmpl-{chat_id}",
+                "choices": [{"delta": {"content": "", "role": "assistant", "function_call": None, "tool_calls": None, "reasoning_content": ""}, "finish_reason": None, "index": 0, "logprobs": None}],
+                "created": int(time.time()),
+                "model": "model",
+                "object": "chat.completion.chunk",
+                "system_fingerprint": "",
+                "usage": None,
+            }
+
+            try:
+                for ans in chat(dia, msg, True, toolcall_session=toolcall_session, tools=tools):
+                    answer = ans["answer"]
+
+                    reasoning_match = re.search(r"<think>(.*?)</think>", answer, flags=re.DOTALL)
+                    if reasoning_match:
+                        reasoning_part = reasoning_match.group(1)
+                        content_part = answer[reasoning_match.end() :]
+                    else:
+                        reasoning_part = ""
+                        content_part = answer
+
+                    reasoning_incremental = ""
+                    if reasoning_part:
+                        if reasoning_part.startswith(reasoning_cache):
+                            reasoning_incremental = reasoning_part.replace(reasoning_cache, "", 1)
+                        else:
+                            reasoning_incremental = reasoning_part
+                        reasoning_cache = reasoning_part
+
+                    content_incremental = ""
+                    if content_part:
+                        if content_part.startswith(answer_cache):
+                            content_incremental = content_part.replace(answer_cache, "", 1)
+                    else:
+                        content_incremental = content_part
+                    answer_cache = content_part
+
+                    token_used += len(reasoning_incremental) + len(content_incremental)
+
+                    if not any([reasoning_incremental, content_incremental]):
+                        continue
+
+                    if reasoning_incremental:
+                        response["choices"][0]["delta"]["reasoning_content"] = reasoning_incremental
+                    else:
+                        response["choices"][0]["delta"]["reasoning_content"] = None
+
+                    if content_incremental:
+                        response["choices"][0]["delta"]["content"] = content_incremental
+                    else:
+                        response["choices"][0]["delta"]["content"] = None
+
+                    yield f"data:{json.dumps(response, ensure_ascii=False)}\n\n"
+            except Exception as e:
+                response["choices"][0]["delta"]["content"] = "**ERROR**: " + str(e)
+                yield f"data:{json.dumps(response, ensure_ascii=False)}\n\n"
+
+            # The last chunk
+            response["choices"][0]["delta"]["content"] = None
+            response["choices"][0]["delta"]["reasoning_content"] = None
+            response["choices"][0]["finish_reason"] = "stop"
+            response["usage"] = {"prompt_tokens": len(prompt), "completion_tokens": token_used, "total_tokens": len(prompt) + token_used}
+            yield f"data:{json.dumps(response, ensure_ascii=False)}\n\n"
+            yield "data:[DONE]\n\n"
+
+        resp = Response(streamed_response_generator(chat_id, dia, msg), mimetype="text/event-stream")
+        resp.headers.add_header("Cache-control", "no-cache")
+        resp.headers.add_header("Connection", "keep-alive")
+        resp.headers.add_header("X-Accel-Buffering", "no")
+        resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
+        return resp
+    else:
+        answer = None
+        for ans in chat(dia, msg, False, toolcall_session=toolcall_session, tools=tools):
+            # focus answer content only
+            answer = ans
+            break
+        content = answer["answer"]
+
+        response = {
+            "id": f"chatcmpl-{chat_id}",
+            "object": "chat.completion",
+            "created": int(time.time()),
+            "model": req.get("model", ""),
+            "usage": {
+                "prompt_tokens": len(prompt),
+                "completion_tokens": len(content),
+                "total_tokens": len(prompt) + len(content),
+                "completion_tokens_details": {
+                    "reasoning_tokens": context_token_used,
+                    "accepted_prediction_tokens": len(content),
+                    "rejected_prediction_tokens": 0,  # 0 for simplicity
+                },
+            },
+            "choices": [{"message": {"role": "assistant", "content": content}, "logprobs": None, "finish_reason": "stop", "index": 0}],
+        }
+        return jsonify(response)
+
+@manager.route('/agents_openai/<agent_id>/chat/completions', methods=['POST'])  # noqa: F821
+@validate_request("model", "messages")  # noqa: F821
+@token_required
+def agents_completion_openai_compatibility (tenant_id, agent_id):
+    req = request.json
+    tiktokenenc = tiktoken.get_encoding("cl100k_base")
+    messages = req.get("messages", [])
+    if not messages:
+        return get_error_data_result("You must provide at least one message.")
+    if not UserCanvasService.query(user_id=tenant_id, id=agent_id):
+        return get_error_data_result(f"You don't own the agent {agent_id}")
+  
+    filtered_messages = [m for m in messages if m["role"] in ["user", "assistant"]]
+    prompt_tokens = sum(len(tiktokenenc.encode(m["content"])) for m in filtered_messages)
+    if not filtered_messages:
+        return jsonify(get_data_openai( 
+            id=agent_id,
+            content="No valid messages found (user or assistant).",
+            finish_reason="stop",
+            model=req.get("model", ""), 
+            completion_tokens=len(tiktokenenc.encode("No valid messages found (user or assistant).")),
+            prompt_tokens=prompt_tokens, 
+        ))
+    
+    # Get the last user message as the question
+    question = next((m["content"] for m in reversed(messages) if m["role"] == "user"), "")
+    
+    if req.get("stream", True):
+        return Response(completionOpenAI(tenant_id, agent_id, question, session_id=req.get("id", req.get("metadata", {}).get("id","")), stream=True), mimetype="text/event-stream")
+    else:
+        # For non-streaming, just return the response directly
+        response = next(completionOpenAI(tenant_id, agent_id, question, session_id=req.get("id", req.get("metadata", {}).get("id","")), stream=False))
+        return jsonify(response)
+    
+
+@manager.route("/agents/<agent_id>/completions", methods=["POST"])  # noqa: F821
 @token_required
 def agent_completions(tenant_id, agent_id):
     req = request.json
@@ -170,12 +406,20 @@ def agent_completions(tenant_id, agent_id):
         dsl = cvs[0].dsl
         if not isinstance(dsl, str):
             dsl = json.dumps(dsl)
-        #canvas = Canvas(dsl, tenant_id)
-        #if canvas.get_preset_param():
-        #    req["question"] = ""
+
         conv = API4ConversationService.query(id=req["session_id"], dialog_id=agent_id)
         if not conv:
             return get_error_data_result(f"You don't own the session {req['session_id']}")
+        # If an update to UserCanvas is detected, update the API4Conversation.dsl
+        sync_dsl = req.get("sync_dsl", False)
+        if sync_dsl is True and cvs[0].update_time > conv[0].update_time:
+            current_dsl = conv[0].dsl
+            new_dsl = json.loads(dsl)
+            state_fields = ["history", "messages", "path", "reference"]
+            states = {field: current_dsl.get(field, []) for field in state_fields}
+            current_dsl.update(new_dsl)
+            current_dsl.update(states)
+            API4ConversationService.update_by_id(req["session_id"], {"dsl": current_dsl})
     else:
         req["question"] = ""
     if req.get("stream", True):
@@ -192,7 +436,7 @@ def agent_completions(tenant_id, agent_id):
         return get_error_data_result(str(e))
 
 
-@manager.route('/chats/<chat_id>/sessions', methods=['GET'])  # noqa: F821
+@manager.route("/chats/<chat_id>/sessions", methods=["GET"])  # noqa: F821
 @token_required
 def list_session(tenant_id, chat_id):
     if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
@@ -211,7 +455,7 @@ def list_session(tenant_id, chat_id):
     if not convs:
         return get_result(data=[])
     for conv in convs:
-        conv['messages'] = conv.pop("message")
+        conv["messages"] = conv.pop("message")
         infos = conv["messages"]
         for info in infos:
             if "prompt" in info:
@@ -220,12 +464,11 @@ def list_session(tenant_id, chat_id):
         if conv["reference"]:
             messages = conv["messages"]
             message_num = 0
-            chunk_num = 0
-            while message_num < len(messages):
+            while message_num < len(messages) and message_num < len(conv["reference"]):
                 if message_num != 0 and messages[message_num]["role"] != "user":
                     chunk_list = []
-                    if "chunks" in conv["reference"][chunk_num]:
-                        chunks = conv["reference"][chunk_num]["chunks"]
+                    if "chunks" in conv["reference"][message_num]:
+                        chunks = conv["reference"][message_num]["chunks"]
                         for chunk in chunks:
                             new_chunk = {
                                 "id": chunk.get("chunk_id", chunk.get("id")),
@@ -238,14 +481,13 @@ def list_session(tenant_id, chat_id):
                             }
 
                             chunk_list.append(new_chunk)
-                    chunk_num += 1
                     messages[message_num]["reference"] = chunk_list
                 message_num += 1
         del conv["reference"]
     return get_result(data=convs)
 
 
-@manager.route('/agents/<agent_id>/sessions', methods=['GET'])  # noqa: F821
+@manager.route("/agents/<agent_id>/sessions", methods=["GET"])  # noqa: F821
 @token_required
 def list_agent_session(tenant_id, agent_id):
     if not UserCanvasService.query(user_id=tenant_id, id=agent_id):
@@ -259,11 +501,13 @@ def list_agent_session(tenant_id, agent_id):
         desc = False
     else:
         desc = True
-    convs = API4ConversationService.get_list(agent_id, tenant_id, page_number, items_per_page, orderby, desc, id, user_id)
+    # dsl defaults to True in all cases except for False and false
+    include_dsl = request.args.get("dsl") != "False" and request.args.get("dsl") != "false"
+    convs = API4ConversationService.get_list(agent_id, tenant_id, page_number, items_per_page, orderby, desc, id, user_id, include_dsl)
     if not convs:
         return get_result(data=[])
     for conv in convs:
-        conv['messages'] = conv.pop("message")
+        conv["messages"] = conv.pop("message")
         infos = conv["messages"]
         for info in infos:
             if "prompt" in info:
@@ -296,11 +540,14 @@ def list_agent_session(tenant_id, agent_id):
     return get_result(data=convs)
 
 
-@manager.route('/chats/<chat_id>/sessions', methods=["DELETE"])  # noqa: F821
+@manager.route("/chats/<chat_id>/sessions", methods=["DELETE"])  # noqa: F821
 @token_required
 def delete(tenant_id, chat_id):
     if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
         return get_error_data_result(message="You don't own the chat")
+    
+    errors = []
+    success_count = 0
     req = request.json
     convs = ConversationService.query(dialog_id=chat_id)
     if not req:
@@ -314,15 +561,98 @@ def delete(tenant_id, chat_id):
             conv_list.append(conv.id)
     else:
         conv_list = ids
+    
+    unique_conv_ids, duplicate_messages = check_duplicate_ids(conv_list, "session")
+    conv_list = unique_conv_ids
+    
     for id in conv_list:
         conv = ConversationService.query(id=id, dialog_id=chat_id)
         if not conv:
-            return get_error_data_result(message="The chat doesn't own the session")
+            errors.append(f"The chat doesn't own the session {id}")
+            continue
         ConversationService.delete_by_id(id)
+        success_count += 1
+    
+    if errors:
+        if success_count > 0:
+            return get_result(
+                data={"success_count": success_count, "errors": errors},
+                message=f"Partially deleted {success_count} sessions with {len(errors)} errors"
+            )
+        else:
+            return get_error_data_result(message="; ".join(errors))
+    
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(
+                message=f"Partially deleted {success_count} sessions with {len(duplicate_messages)} errors", 
+                data={"success_count": success_count, "errors": duplicate_messages}
+            )
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+    
     return get_result()
 
 
-@manager.route('/sessions/ask', methods=['POST'])  # noqa: F821
+@manager.route("/agents/<agent_id>/sessions", methods=["DELETE"])  # noqa: F821
+@token_required
+def delete_agent_session(tenant_id, agent_id):
+    errors = []
+    success_count = 0
+    req = request.json
+    cvs = UserCanvasService.query(user_id=tenant_id, id=agent_id)
+    if not cvs:
+        return get_error_data_result(f"You don't own the agent {agent_id}")
+
+    convs = API4ConversationService.query(dialog_id=agent_id)
+    if not convs:
+        return get_error_data_result(f"Agent {agent_id} has no sessions")
+
+    if not req:
+        ids = None
+    else:
+        ids = req.get("ids")
+
+    if not ids:
+        conv_list = []
+        for conv in convs:
+            conv_list.append(conv.id)
+    else:
+        conv_list = ids
+
+    unique_conv_ids, duplicate_messages = check_duplicate_ids(conv_list, "session")
+    conv_list = unique_conv_ids
+
+    for session_id in conv_list:
+        conv = API4ConversationService.query(id=session_id, dialog_id=agent_id)
+        if not conv:
+            errors.append(f"The agent doesn't own the session {session_id}")
+            continue
+        API4ConversationService.delete_by_id(session_id)
+        success_count += 1
+    
+    if errors:
+        if success_count > 0:
+            return get_result(
+                data={"success_count": success_count, "errors": errors},
+                message=f"Partially deleted {success_count} sessions with {len(errors)} errors"
+            )
+        else:
+            return get_error_data_result(message="; ".join(errors))
+    
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(
+                message=f"Partially deleted {success_count} sessions with {len(duplicate_messages)} errors", 
+                data={"success_count": success_count, "errors": duplicate_messages}
+            )
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+    
+    return get_result()
+
+
+@manager.route("/sessions/ask", methods=["POST"])  # noqa: F821
 @token_required
 def ask_about(tenant_id):
     req = request.json
@@ -348,9 +678,7 @@ def ask_about(tenant_id):
             for ans in ask(req["question"], req["kb_ids"], uid):
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
         except Exception as e:
-            yield "data:" + json.dumps({"code": 500, "message": str(e),
-                                        "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
-                                       ensure_ascii=False) + "\n\n"
+            yield "data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}}, ensure_ascii=False) + "\n\n"
         yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
 
     resp = Response(stream(), mimetype="text/event-stream")
@@ -361,7 +689,7 @@ def ask_about(tenant_id):
     return resp
 
 
-@manager.route('/sessions/related_questions', methods=['POST'])  # noqa: F821
+@manager.route("/sessions/related_questions", methods=["POST"])  # noqa: F821
 @token_required
 def related_questions(tenant_id):
     req = request.json
@@ -393,18 +721,27 @@ Reason:
  - At the same time, related terms can also help search engines better understand user needs and return more accurate search results.
 
 """
-    ans = chat_mdl.chat(prompt, [{"role": "user", "content": f"""
+    ans = chat_mdl.chat(
+        prompt,
+        [
+            {
+                "role": "user",
+                "content": f"""
 Keywords: {question}
 Related search terms:
-    """}], {"temperature": 0.9})
+    """,
+            }
+        ],
+        {"temperature": 0.9},
+    )
     return get_result(data=[re.sub(r"^[0-9]\. ", "", a) for a in ans.split("\n") if re.match(r"^[0-9]\. ", a)])
 
 
-@manager.route('/chatbots/<dialog_id>/completions', methods=['POST'])  # noqa: F821
+@manager.route("/chatbots/<dialog_id>/completions", methods=["POST"])  # noqa: F821
 def chatbot_completions(dialog_id):
     req = request.json
 
-    token = request.headers.get('Authorization').split()
+    token = request.headers.get("Authorization").split()
     if len(token) != 2:
         return get_error_data_result(message='Authorization is not valid!"')
     token = token[1]
@@ -427,11 +764,11 @@ def chatbot_completions(dialog_id):
         return get_result(data=answer)
 
 
-@manager.route('/agentbots/<agent_id>/completions', methods=['POST'])  # noqa: F821
+@manager.route("/agentbots/<agent_id>/completions", methods=["POST"])  # noqa: F821
 def agent_bot_completions(agent_id):
     req = request.json
 
-    token = request.headers.get('Authorization').split()
+    token = request.headers.get("Authorization").split()
     if len(token) != 2:
         return get_error_data_result(message='Authorization is not valid!"')
     token = token[1]

api/apps/search_app.py

@@ -0,0 +1,188 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from flask import request
+from flask_login import current_user, login_required
+
+from api import settings
+from api.constants import DATASET_NAME_LIMIT
+from api.db import StatusEnum
+from api.db.db_models import DB
+from api.db.services import duplicate_name
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.search_service import SearchService
+from api.db.services.user_service import TenantService, UserTenantService
+from api.utils import get_uuid
+from api.utils.api_utils import get_data_error_result, get_json_result, not_allowed_parameters, server_error_response, validate_request
+
+
+@manager.route("/create", methods=["post"])  # noqa: F821
+@login_required
+@validate_request("name")
+def create():
+    req = request.get_json()
+    search_name = req["name"]
+    description = req.get("description", "")
+    if not isinstance(search_name, str):
+        return get_data_error_result(message="Search name must be string.")
+    if search_name.strip() == "":
+        return get_data_error_result(message="Search name can't be empty.")
+    if len(search_name.encode("utf-8")) > DATASET_NAME_LIMIT:
+        return get_data_error_result(message=f"Search name length is {len(search_name)} which is large than {DATASET_NAME_LIMIT}")
+    e, _ = TenantService.get_by_id(current_user.id)
+    if not e:
+        return get_data_error_result(message="Authorizationd identity.")
+
+    search_name = search_name.strip()
+    search_name = duplicate_name(KnowledgebaseService.query, name=search_name, tenant_id=current_user.id, status=StatusEnum.VALID.value)
+
+    req["id"] = get_uuid()
+    req["name"] = search_name
+    req["description"] = description
+    req["tenant_id"] = current_user.id
+    req["created_by"] = current_user.id
+    with DB.atomic():
+        try:
+            if not SearchService.save(**req):
+                return get_data_error_result()
+            return get_json_result(data={"search_id": req["id"]})
+        except Exception as e:
+            return server_error_response(e)
+
+
+@manager.route("/update", methods=["post"])  # noqa: F821
+@login_required
+@validate_request("search_id", "name", "search_config", "tenant_id")
+@not_allowed_parameters("id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by")
+def update():
+    req = request.get_json()
+    if not isinstance(req["name"], str):
+        return get_data_error_result(message="Search name must be string.")
+    if req["name"].strip() == "":
+        return get_data_error_result(message="Search name can't be empty.")
+    if len(req["name"].encode("utf-8")) > DATASET_NAME_LIMIT:
+        return get_data_error_result(message=f"Search name length is {len(req['name'])} which is large than {DATASET_NAME_LIMIT}")
+    req["name"] = req["name"].strip()
+    tenant_id = req["tenant_id"]
+    e, _ = TenantService.get_by_id(tenant_id)
+    if not e:
+        return get_data_error_result(message="Authorizationd identity.")
+
+    search_id = req["search_id"]
+    if not SearchService.accessible4deletion(search_id, current_user.id):
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+
+    try:
+        search_app = SearchService.query(tenant_id=tenant_id, id=search_id)[0]
+        if not search_app:
+            return get_json_result(data=False, message=f"Cannot find search {search_id}", code=settings.RetCode.DATA_ERROR)
+
+        if req["name"].lower() != search_app.name.lower() and len(SearchService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) >= 1:
+            return get_data_error_result(message="Duplicated search name.")
+
+        if "search_config" in req:
+            current_config = search_app.search_config or {}
+            new_config = req["search_config"]
+
+            if not isinstance(new_config, dict):
+                return get_data_error_result(message="search_config must be a JSON object")
+
+            updated_config = {**current_config, **new_config}
+            req["search_config"] = updated_config
+
+        req.pop("search_id", None)
+        req.pop("tenant_id", None)
+
+        updated = SearchService.update_by_id(search_id, req)
+        if not updated:
+            return get_data_error_result(message="Failed to update search")
+
+        e, updated_search = SearchService.get_by_id(search_id)
+        if not e:
+            return get_data_error_result(message="Failed to fetch updated search")
+
+        return get_json_result(data=updated_search.to_dict())
+
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/detail", methods=["GET"])  # noqa: F821
+@login_required
+def detail():
+    search_id = request.args["search_id"]
+    try:
+        tenants = UserTenantService.query(user_id=current_user.id)
+        for tenant in tenants:
+            if SearchService.query(tenant_id=tenant.tenant_id, id=search_id):
+                break
+        else:
+            return get_json_result(data=False, message="Has no permission for this operation.", code=settings.RetCode.OPERATING_ERROR)
+
+        search = SearchService.get_detail(search_id)
+        if not search:
+            return get_data_error_result(message="Can't find this Search App!")
+        return get_json_result(data=search)
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/list", methods=["POST"])  # noqa: F821
+@login_required
+def list_search_app():
+    keywords = request.args.get("keywords", "")
+    page_number = int(request.args.get("page", 0))
+    items_per_page = int(request.args.get("page_size", 0))
+    orderby = request.args.get("orderby", "create_time")
+    if request.args.get("desc", "true").lower() == "false":
+        desc = False
+    else:
+        desc = True
+
+    req = request.get_json()
+    owner_ids = req.get("owner_ids", [])
+    try:
+        if not owner_ids:
+            tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
+            tenants = [m["tenant_id"] for m in tenants]
+            search_apps, total = SearchService.get_by_tenant_ids(tenants, current_user.id, page_number, items_per_page, orderby, desc, keywords)
+        else:
+            tenants = owner_ids
+            search_apps, total = SearchService.get_by_tenant_ids(tenants, current_user.id, 0, 0, orderby, desc, keywords)
+            search_apps = [search_app for search_app in search_apps if search_app["tenant_id"] in tenants]
+            total = len(search_apps)
+            if page_number and items_per_page:
+                search_apps = search_apps[(page_number - 1) * items_per_page : page_number * items_per_page]
+        return get_json_result(data={"search_apps": search_apps, "total": total})
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/rm", methods=["post"])  # noqa: F821
+@login_required
+@validate_request("search_id")
+def rm():
+    req = request.get_json()
+    search_id = req["search_id"]
+    if not SearchService.accessible4deletion(search_id, current_user.id):
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+
+    try:
+        if not SearchService.delete_by_id(search_id):
+            return get_data_error_result(message=f"Failed to delete search App {search_id}")
+        return get_json_result(data=True)
+    except Exception as e:
+        return server_error_response(e)

api/apps/system_app.py

@@ -37,7 +37,6 @@ from timeit import default_timer as timer
 
 from rag.utils.redis_conn import REDIS_CONN
 
-
 @manager.route("/version", methods=["GET"])  # noqa: F821
 @login_required
 def version():
@@ -201,7 +200,7 @@ def new_token():
         if not tenants:
             return get_data_error_result(message="Tenant not found!")
 
-        tenant_id = tenants[0].tenant_id
+        tenant_id = [tenant for tenant in tenants if tenant.role == 'owner'][0].tenant_id
         obj = {
             "tenant_id": tenant_id,
             "token": generate_confirmation_token(tenant_id),
@@ -256,7 +255,7 @@ def token_list():
         if not tenants:
             return get_data_error_result(message="Tenant not found!")
 
-        tenant_id = tenants[0].tenant_id
+        tenant_id = [tenant for tenant in tenants if tenant.role == 'owner'][0].tenant_id
         objs = APITokenService.query(tenant_id=tenant_id)
         objs = [o.to_dict() for o in objs]
         for o in objs:
@@ -298,3 +297,25 @@ def rm(token):
         [APIToken.tenant_id == current_user.id, APIToken.token == token]
     )
     return get_json_result(data=True)
+
+
+@manager.route('/config', methods=['GET'])  # noqa: F821
+def get_config():
+    """
+    Get system configuration.
+    ---
+    tags:
+        - System
+    responses:
+        200:
+            description: Return system configuration
+            schema:
+                type: object
+                properties:
+                    registerEnable:
+                        type: integer 0 means disabled, 1 means enabled
+                        description: Whether user registration is enabled
+    """
+    return get_json_result(data={
+        "registerEnabled": settings.REGISTER_ENABLED
+    })

api/apps/user_app.py

@@ -13,35 +13,38 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import logging
 import json
+import logging
 import re
+import secrets
 from datetime import datetime
 
-from flask import request, session, redirect
-from werkzeug.security import generate_password_hash, check_password_hash
-from flask_login import login_required, current_user, login_user, logout_user
+from flask import redirect, request, session
+from flask_login import current_user, login_required, login_user, logout_user
+from werkzeug.security import check_password_hash, generate_password_hash
 
+from api import settings
+from api.apps.auth import get_auth_client
+from api.db import FileType, UserTenantRole
 from api.db.db_models import TenantLLM
-from api.db.services.llm_service import TenantLLMService, LLMService
-from api.utils.api_utils import (
-    server_error_response,
-    validate_request,
-    get_data_error_result,
-)
+from api.db.services.file_service import FileService
+from api.db.services.llm_service import LLMService, TenantLLMService
+from api.db.services.user_service import TenantService, UserService, UserTenantService
 from api.utils import (
-    get_uuid,
-    get_format_time,
-    decrypt,
-    download_img,
     current_timestamp,
     datetime_format,
+    decrypt,
+    download_img,
+    get_format_time,
+    get_uuid,
+)
+from api.utils.api_utils import (
+    construct_response,
+    get_data_error_result,
+    get_json_result,
+    server_error_response,
+    validate_request,
 )
-from api.db import UserTenantRole, FileType
-from api import settings
-from api.db.services.user_service import UserService, TenantService, UserTenantService
-from api.db.services.file_service import FileService
-from api.utils.api_utils import get_json_result, construct_response
 
 
 @manager.route("/login", methods=["POST", "GET"])  # noqa: F821
@@ -76,9 +79,7 @@ def login():
           type: object
     """
     if not request.json:
-        return get_json_result(
-            data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="Unauthorized!"
-        )
+        return get_json_result(data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="Unauthorized!")
 
     email = request.json.get("email", "")
     users = UserService.query(email=email)
@@ -93,9 +94,7 @@ def login():
     try:
         password = decrypt(password)
     except BaseException:
-        return get_json_result(
-            data=False, code=settings.RetCode.SERVER_ERROR, message="Fail to crypt password"
-        )
+        return get_json_result(data=False, code=settings.RetCode.SERVER_ERROR, message="Fail to crypt password")
 
     user = UserService.query_user(email, password)
     if user:
@@ -115,9 +114,131 @@ def login():
         )
 
 
+@manager.route("/login/channels", methods=["GET"])  # noqa: F821
+def get_login_channels():
+    """
+    Get all supported authentication channels.
+    """
+    try:
+        channels = []
+        for channel, config in settings.OAUTH_CONFIG.items():
+            channels.append(
+                {
+                    "channel": channel,
+                    "display_name": config.get("display_name", channel.title()),
+                    "icon": config.get("icon", "sso"),
+                }
+            )
+        return get_json_result(data=channels)
+    except Exception as e:
+        logging.exception(e)
+        return get_json_result(data=[], message=f"Load channels failure, error: {str(e)}", code=settings.RetCode.EXCEPTION_ERROR)
+
+
+@manager.route("/login/<channel>", methods=["GET"])  # noqa: F821
+def oauth_login(channel):
+    channel_config = settings.OAUTH_CONFIG.get(channel)
+    if not channel_config:
+        raise ValueError(f"Invalid channel name: {channel}")
+    auth_cli = get_auth_client(channel_config)
+
+    state = get_uuid()
+    session["oauth_state"] = state
+    auth_url = auth_cli.get_authorization_url(state)
+    return redirect(auth_url)
+
+
+@manager.route("/oauth/callback/<channel>", methods=["GET"])  # noqa: F821
+def oauth_callback(channel):
+    """
+    Handle the OAuth/OIDC callback for various channels dynamically.
+    """
+    try:
+        channel_config = settings.OAUTH_CONFIG.get(channel)
+        if not channel_config:
+            raise ValueError(f"Invalid channel name: {channel}")
+        auth_cli = get_auth_client(channel_config)
+
+        # Check the state
+        state = request.args.get("state")
+        if not state or state != session.get("oauth_state"):
+            return redirect("/?error=invalid_state")
+        session.pop("oauth_state", None)
+
+        # Obtain the authorization code
+        code = request.args.get("code")
+        if not code:
+            return redirect("/?error=missing_code")
+
+        # Exchange authorization code for access token
+        token_info = auth_cli.exchange_code_for_token(code)
+        access_token = token_info.get("access_token")
+        if not access_token:
+            return redirect("/?error=token_failed")
+
+        id_token = token_info.get("id_token")
+
+        # Fetch user info
+        user_info = auth_cli.fetch_user_info(access_token, id_token=id_token)
+        if not user_info.email:
+            return redirect("/?error=email_missing")
+
+        # Login or register
+        users = UserService.query(email=user_info.email)
+        user_id = get_uuid()
+
+        if not users:
+            try:
+                try:
+                    avatar = download_img(user_info.avatar_url)
+                except Exception as e:
+                    logging.exception(e)
+                    avatar = ""
+
+                users = user_register(
+                    user_id,
+                    {
+                        "access_token": get_uuid(),
+                        "email": user_info.email,
+                        "avatar": avatar,
+                        "nickname": user_info.nickname,
+                        "login_channel": channel,
+                        "last_login_time": get_format_time(),
+                        "is_superuser": False,
+                    },
+                )
+
+                if not users:
+                    raise Exception(f"Failed to register {user_info.email}")
+                if len(users) > 1:
+                    raise Exception(f"Same email: {user_info.email} exists!")
+
+                # Try to log in
+                user = users[0]
+                login_user(user)
+                return redirect(f"/?auth={user.get_id()}")
+
+            except Exception as e:
+                rollback_user_registration(user_id)
+                logging.exception(e)
+                return redirect(f"/?error={str(e)}")
+
+        # User exists, try to log in
+        user = users[0]
+        user.access_token = get_uuid()
+        login_user(user)
+        user.save()
+        return redirect(f"/?auth={user.get_id()}")
+    except Exception as e:
+        logging.exception(e)
+        return redirect(f"/?error={str(e)}")
+
+
 @manager.route("/github_callback", methods=["GET"])  # noqa: F821
 def github_callback():
     """
+    **Deprecated**, Use `/oauth/callback/<channel>` instead.
+
     GitHub OAuth callback endpoint.
     ---
     tags:
@@ -309,9 +430,7 @@ def user_info_from_feishu(access_token):
         "Content-Type": "application/json; charset=utf-8",
         "Authorization": f"Bearer {access_token}",
     }
-    res = requests.get(
-        "https://open.feishu.cn/open-apis/authen/v1/user_info", headers=headers
-    )
+    res = requests.get("https://open.feishu.cn/open-apis/authen/v1/user_info", headers=headers)
     user_info = res.json()["data"]
     user_info["email"] = None if user_info.get("email") == "" else user_info["email"]
     return user_info
@@ -321,17 +440,13 @@ def user_info_from_github(access_token):
     import requests
 
     headers = {"Accept": "application/json", "Authorization": f"token {access_token}"}
-    res = requests.get(
-        f"https://api.github.com/user?access_token={access_token}", headers=headers
-    )
+    res = requests.get(f"https://api.github.com/user?access_token={access_token}", headers=headers)
     user_info = res.json()
     email_info = requests.get(
         f"https://api.github.com/user/emails?access_token={access_token}",
         headers=headers,
     ).json()
-    user_info["email"] = next(
-        (email for email in email_info if email["primary"]), None
-    )["email"]
+    user_info["email"] = next((email for email in email_info if email["primary"]), None)["email"]
     return user_info
 
 
@@ -351,7 +466,7 @@ def log_out():
         schema:
           type: object
     """
-    current_user.access_token = ""
+    current_user.access_token = f"INVALID_{secrets.token_hex(16)}"
     current_user.save()
     logout_user()
     return get_json_result(data=True)
@@ -391,9 +506,7 @@ def setting_user():
     request_data = request.json
     if request_data.get("password"):
         new_password = request_data.get("new_password")
-        if not check_password_hash(
-                current_user.password, decrypt(request_data["password"])
-        ):
+        if not check_password_hash(current_user.password, decrypt(request_data["password"])):
             return get_json_result(
                 data=False,
                 code=settings.RetCode.AUTHENTICATION_ERROR,
@@ -424,9 +537,7 @@ def setting_user():
         return get_json_result(data=True)
     except Exception as e:
         logging.exception(e)
-        return get_json_result(
-            data=False, message="Update failure!", code=settings.RetCode.EXCEPTION_ERROR
-        )
+        return get_json_result(data=False, message="Update failure!", code=settings.RetCode.EXCEPTION_ERROR)
 
 
 @manager.route("/info", methods=["GET"])  # noqa: F821
@@ -518,9 +629,23 @@ def user_register(user_id, user):
                 "model_type": llm.model_type,
                 "api_key": settings.API_KEY,
                 "api_base": settings.LLM_BASE_URL,
-                "max_tokens": llm.max_tokens if llm.max_tokens else 8192
+                "max_tokens": llm.max_tokens if llm.max_tokens else 8192,
             }
         )
+    if settings.LIGHTEN != 1:
+        for buildin_embedding_model in settings.BUILTIN_EMBEDDING_MODELS:
+            mdlnm, fid = TenantLLMService.split_model_name_and_factory(buildin_embedding_model)
+            tenant_llm.append(
+                {
+                    "tenant_id": user_id,
+                    "llm_factory": fid,
+                    "llm_name": mdlnm,
+                    "model_type": "embedding",
+                    "api_key": "",
+                    "api_base": "",
+                    "max_tokens": 1024 if buildin_embedding_model == "BAAI/bge-large-zh-v1.5@BAAI" else 512,
+                }
+            )
 
     if not UserService.save(**user):
         return
@@ -562,11 +687,19 @@ def user_add():
         schema:
           type: object
     """
+
+    if not settings.REGISTER_ENABLED:
+        return get_json_result(
+            data=False,
+            message="User registration is disabled!",
+            code=settings.RetCode.OPERATING_ERROR,
+        )
+
     req = request.json
     email_address = req["email"]
 
     # Validate the email address
-    if not re.match(r"^[\w\._-]+@([\w_-]+\.)+[\w-]{2,5}$", email_address):
+    if not re.match(r"^[\w\._-]+@([\w_-]+\.)+[\w-]{2,}$", email_address):
         return get_json_result(
             data=False,
             message=f"Invalid email address: {email_address}!",

api/constants.py

@@ -13,9 +13,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-NAME_LENGTH_LIMIT = 2 ** 10
+NAME_LENGTH_LIMIT = 2**10
 
-IMG_BASE64_PREFIX = 'data:image/png;base64,'
+IMG_BASE64_PREFIX = "data:image/png;base64,"
 
 SERVICE_CONF = "service_conf.yaml"
 
@@ -25,3 +25,4 @@ REQUEST_WAIT_SEC = 2
 REQUEST_MAX_WAIT_SEC = 300
 
 DATASET_NAME_LIMIT = 128
+FILE_NAME_LEN_LIMIT = 255

api/db/__init__.py

@@ -49,6 +49,7 @@ class FileType(StrEnum):
     FOLDER = 'folder'
     OTHER = "other"
 
+VALID_FILE_TYPES = {FileType.PDF, FileType.DOC, FileType.VISUAL, FileType.AURAL, FileType.VIRTUAL, FileType.FOLDER, FileType.OTHER}
 
 class LLMType(StrEnum):
     CHAT = 'chat'
@@ -73,6 +74,7 @@ class TaskStatus(StrEnum):
     DONE = "3"
     FAIL = "4"
 
+VALID_TASK_STATUS     = {TaskStatus.UNSTART, TaskStatus.RUNNING, TaskStatus.CANCEL, TaskStatus.DONE, TaskStatus.FAIL}
 
 class ParserType(StrEnum):
     PRESENTATION = "presentation"

api/db/init_data.py

@@ -84,14 +84,14 @@ def init_superuser():
         {"role": "user", "content": "Hello!"}], gen_conf={})
     if msg.find("ERROR: ") == 0:
         logging.error(
-            "'{}' dosen't work. {}".format(
+            "'{}' doesn't work. {}".format(
                 tenant["llm_id"],
                 msg))
     embd_mdl = LLMBundle(tenant["id"], LLMType.EMBEDDING, tenant["embd_id"])
     v, c = embd_mdl.encode(["Hello!"])
     if c == 0:
         logging.error(
-            "'{}' dosen't work!".format(
+            "'{}' doesn't work!".format(
                 tenant["embd_id"]))
 
 
@@ -103,16 +103,12 @@ def init_llm_factory():
     except Exception:
         pass
 
-    factory_llm_infos = json.load(
-        open(
-            os.path.join(get_project_base_directory(), "conf", "llm_factories.json"),
-            "r",
-        )
-    )
-    for factory_llm_info in factory_llm_infos["factory_llm_infos"]:
-        llm_infos = factory_llm_info.pop("llm")
+    factory_llm_infos = settings.FACTORY_LLM_INFOS    
+    for factory_llm_info in factory_llm_infos:
+        info = deepcopy(factory_llm_info)
+        llm_infos = info.pop("llm")
         try:
-            LLMFactoriesService.save(**factory_llm_info)
+            LLMFactoriesService.save(**info)
         except Exception:
             pass
         LLMService.filter_delete([LLM.fid == factory_llm_info["name"]])
@@ -123,7 +119,7 @@ def init_llm_factory():
             except Exception:
                 pass
 
-    LLMFactoriesService.filter_delete([LLMFactories.name == "Local"])
+    LLMFactoriesService.filter_delete([(LLMFactories.name == "Local") | (LLMFactories.name == "novita.ai")])
     LLMService.filter_delete([LLM.fid == "Local"])
     LLMService.filter_delete([LLM.llm_name == "qwen-vl-max"])
     LLMService.filter_delete([LLM.fid == "Moonshot", LLM.llm_name == "flag-embedding"])
@@ -152,7 +148,7 @@ def init_llm_factory():
                 pass
             break
     for kb_id in KnowledgebaseService.get_all_ids():
-        KnowledgebaseService.update_by_id(kb_id, {"doc_num": DocumentService.get_kb_doc_count(kb_id)})
+        KnowledgebaseService.update_document_number_in_init(kb_id=kb_id, doc_num=DocumentService.get_kb_doc_count(kb_id))
 
 
 
@@ -160,7 +156,7 @@ def add_graph_templates():
     dir = os.path.join(get_project_base_directory(), "agent", "templates")
     for fnm in os.listdir(dir):
         try:
-            cnvs = json.load(open(os.path.join(dir, fnm), "r"))
+            cnvs = json.load(open(os.path.join(dir, fnm), "r",encoding="utf-8"))
             try:
                 CanvasTemplateService.save(**cnvs)
             except Exception:

api/db/services/__init__.py

@@ -13,27 +13,87 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import pathlib
 import re
+from pathlib import PurePath
+
 from .user_service import UserService as UserService
 
 
-def duplicate_name(query_func, **kwargs):
-    fnm = kwargs["name"]
-    objs = query_func(**kwargs)
-    if not objs:
-        return fnm
-    ext = pathlib.Path(fnm).suffix #.jpg
-    nm = re.sub(r"%s$"%ext, "", fnm)
-    r = re.search(r"\(([0-9]+)\)$", nm)
-    c = 0
-    if r:
-        c = int(r.group(1))
-        nm = re.sub(r"\([0-9]+\)$", "", nm)
-    c += 1
-    nm = f"{nm}({c})"
-    if ext:
-        nm += f"{ext}"
+def split_name_counter(filename: str) -> tuple[str, int | None]:
+    """
+    Splits a filename into main part and counter (if present in parentheses).
 
-    kwargs["name"] = nm
-    return duplicate_name(query_func, **kwargs)
+    Args:
+        filename: Input filename string to be parsed
+
+    Returns:
+        A tuple containing:
+        - The main filename part (string)
+        - The counter from parentheses (integer) or None if no counter exists
+    """
+    pattern = re.compile(r"^(.*?)\((\d+)\)$")
+
+    match = pattern.search(filename)
+    if match:
+        main_part = match.group(1).rstrip()
+        bracket_part = match.group(2)
+        return main_part, int(bracket_part)
+
+    return filename, None
+
+
+def duplicate_name(query_func, **kwargs) -> str:
+    """
+    Generates a unique filename by appending/incrementing a counter when duplicates exist.
+
+    Continuously checks for name availability using the provided query function,
+    automatically appending (1), (2), etc. until finding an available name or
+    reaching maximum retries.
+
+    Args:
+        query_func: Callable that accepts keyword arguments and returns:
+                  - True if name exists (should be modified)
+                  - False if name is available
+        **kwargs: Must contain 'name' key with original filename to check
+
+    Returns:
+        str: Available filename, either:
+            - Original name (if available)
+            - Modified name with counter (e.g., "file(1).txt")
+
+    Raises:
+        KeyError: If 'name' key not provided in kwargs
+        RuntimeError: If unable to generate unique name after maximum retries
+
+    Example:
+        >>> def name_exists(name): return name in existing_files
+        >>> duplicate_name(name_exists, name="document.pdf")
+        'document(1).pdf'  # If original exists
+    """
+    MAX_RETRIES = 1000
+
+    if "name" not in kwargs:
+        raise KeyError("Arguments must contain 'name' key")
+
+    original_name = kwargs["name"]
+    current_name = original_name
+    retries = 0
+
+    while retries < MAX_RETRIES:
+        if not query_func(**kwargs):
+            return current_name
+
+        path = PurePath(current_name)
+        stem = path.stem
+        suffix = path.suffix
+
+        main_part, counter = split_name_counter(stem)
+        counter = counter + 1 if counter else 1
+
+        new_name = f"{main_part}({counter}){suffix}"
+
+        kwargs["name"] = new_name
+        current_name = new_name
+        retries += 1
+
+    raise RuntimeError(f"Failed to generate unique name within {MAX_RETRIES} attempts. Original: {original_name}")

api/db/services/api_service.py

@@ -43,8 +43,12 @@ class API4ConversationService(CommonService):
     @DB.connection_context()
     def get_list(cls, dialog_id, tenant_id,
                  page_number, items_per_page,
-                 orderby, desc, id, user_id=None):
-        sessions = cls.model.select().where(cls.model.dialog_id == dialog_id)
+                 orderby, desc, id, user_id=None, include_dsl=True):
+        if include_dsl:
+            sessions = cls.model.select().where(cls.model.dialog_id == dialog_id)
+        else:
+            fields = [field for field in cls.model._meta.fields.values() if field.name != 'dsl']
+            sessions = cls.model.select(*fields).where(cls.model.dialog_id == dialog_id)
         if id:
             sessions = sessions.where(cls.model.id == id)
         if user_id:

api/db/services/canvas_service.py

@@ -18,13 +18,15 @@ import time
 import traceback
 from uuid import uuid4
 from agent.canvas import Canvas
-from api.db.db_models import DB, CanvasTemplate, UserCanvas, API4Conversation
+from api.db import TenantPermission
+from api.db.db_models import DB, CanvasTemplate, User, UserCanvas, API4Conversation
 from api.db.services.api_service import API4ConversationService
 from api.db.services.common_service import CommonService
 from api.db.services.conversation_service import structure_answer
 from api.utils import get_uuid
-
-
+from api.utils.api_utils import get_data_openai
+import tiktoken
+from peewee import fn
 class CanvasTemplateService(CommonService):
     model = CanvasTemplate
 
@@ -50,7 +52,74 @@ class UserCanvasService(CommonService):
         agents = agents.paginate(page_number, items_per_page)
 
         return list(agents.dicts())
-
+   
+    @classmethod
+    @DB.connection_context()
+    def get_by_tenant_id(cls, pid):
+        try:
+            
+            fields = [
+                cls.model.id,
+                cls.model.avatar,
+                cls.model.title,
+                cls.model.dsl,
+                cls.model.description,
+                cls.model.permission,
+                cls.model.update_time,
+                cls.model.user_id,
+                cls.model.create_time,
+                cls.model.create_date,
+                cls.model.update_date,
+                User.nickname,
+                User.avatar.alias('tenant_avatar'),
+            ]
+            agents = cls.model.select(*fields) \
+            .join(User, on=(cls.model.user_id == User.id)) \
+            .where(cls.model.id == pid)
+            # obj = cls.model.query(id=pid)[0]
+            return True, agents.dicts()[0]
+        except Exception as e:
+            print(e)
+            return False, None
+     
+    @classmethod
+    @DB.connection_context()
+    def get_by_tenant_ids(cls, joined_tenant_ids, user_id,
+                          page_number, items_per_page,
+                          orderby, desc, keywords,
+                          ):
+        fields = [
+            cls.model.id,
+            cls.model.avatar,
+            cls.model.title,
+            cls.model.dsl,
+            cls.model.description,
+            cls.model.permission,
+            User.nickname,
+            User.avatar.alias('tenant_avatar'),
+            cls.model.update_time
+        ]
+        if keywords:
+            agents = cls.model.select(*fields).join(User, on=(cls.model.user_id == User.id)).where(
+                ((cls.model.user_id.in_(joined_tenant_ids) & (cls.model.permission == 
+                                                                TenantPermission.TEAM.value)) | (
+                    cls.model.user_id == user_id)),
+                (fn.LOWER(cls.model.title).contains(keywords.lower()))
+            )
+        else:
+            agents = cls.model.select(*fields).join(User, on=(cls.model.user_id == User.id)).where(
+                ((cls.model.user_id.in_(joined_tenant_ids) & (cls.model.permission == 
+                                                                TenantPermission.TEAM.value)) | (
+                    cls.model.user_id == user_id))
+            )
+        if desc:
+            agents = agents.order_by(cls.model.getter_by(orderby).desc())
+        else:
+            agents = agents.order_by(cls.model.getter_by(orderby).asc())
+        count = agents.count()
+        agents = agents.paginate(page_number, items_per_page)
+        return list(agents.dicts()), count
+   
 
 def completion(tenant_id, agent_id, question, session_id=None, stream=True, **kwargs):
     e, cvs = UserCanvasService.get_by_id(agent_id)
@@ -86,21 +155,7 @@ def completion(tenant_id, agent_id, question, session_id=None, stream=True, **kw
             "dsl": cvs.dsl
         }
         API4ConversationService.save(**conv)
-        if query:
-            yield "data:" + json.dumps({"code": 0,
-                                        "message": "",
-                                        "data": {
-                                            "session_id": session_id,
-                                            "answer": canvas.get_prologue(),
-                                            "reference": [],
-                                            "param": canvas.get_preset_param()
-                                        }
-                                        },
-                                       ensure_ascii=False) + "\n\n"
-            yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
-            return
-        else:
-            conv = API4Conversation(**conv)
+        conv = API4Conversation(**conv)
     else:
         e, conv = API4ConversationService.get_by_id(session_id)
         assert e, "Session not found!"
@@ -118,6 +173,19 @@ def completion(tenant_id, agent_id, question, session_id=None, stream=True, **kw
             conv.reference = []
         conv.reference.append({"chunks": [], "doc_aggs": []})
 
+        kwargs_changed = False
+        if kwargs:
+            query = canvas.get_preset_param()
+            if query:
+                for ele in query:
+                    if ele["key"] in kwargs:
+                        if ele["value"] != kwargs[ele["key"]]:
+                            ele["value"] = kwargs[ele["key"]]
+                            kwargs_changed = True
+        if kwargs_changed:
+            conv.dsl = json.loads(str(canvas))
+            API4ConversationService.update_by_id(session_id, {"dsl": conv.dsl})
+
     final_ans = {"reference": [], "content": ""}
     if stream:
         try:
@@ -130,7 +198,7 @@ def completion(tenant_id, agent_id, question, session_id=None, stream=True, **kw
                     continue
                 for k in ans.keys():
                     final_ans[k] = ans[k]
-                ans = {"answer": ans["content"], "reference": ans.get("reference", [])}
+                ans = {"answer": ans["content"], "reference": ans.get("reference", []), "param": canvas.get_preset_param()}
                 ans = structure_answer(conv, ans, message_id, session_id)
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans},
                                            ensure_ascii=False) + "\n\n"
@@ -160,8 +228,225 @@ def completion(tenant_id, agent_id, question, session_id=None, stream=True, **kw
                 canvas.reference.append(final_ans["reference"])
             conv.dsl = json.loads(str(canvas))
 
-            result = {"answer": final_ans["content"], "reference": final_ans.get("reference", [])}
+            result = {"answer": final_ans["content"], "reference": final_ans.get("reference", []) , "param": canvas.get_preset_param()}
             result = structure_answer(conv, result, message_id, session_id)
             API4ConversationService.append_message(conv.id, conv.to_dict())
             yield result
-            break
+            break
+def completionOpenAI(tenant_id, agent_id, question, session_id=None, stream=True, **kwargs):
+    """Main function for OpenAI-compatible completions, structured similarly to the completion function."""
+    tiktokenenc = tiktoken.get_encoding("cl100k_base")
+    e, cvs = UserCanvasService.get_by_id(agent_id)
+    
+    if not e:
+        yield get_data_openai(
+            id=session_id,
+            model=agent_id,
+            content="**ERROR**: Agent not found."
+        )
+        return
+    
+    if cvs.user_id != tenant_id:
+        yield get_data_openai(
+            id=session_id,
+            model=agent_id,
+            content="**ERROR**: You do not own the agent"
+        )
+        return
+    
+    if not isinstance(cvs.dsl, str):
+        cvs.dsl = json.dumps(cvs.dsl, ensure_ascii=False)
+    
+    canvas = Canvas(cvs.dsl, tenant_id)
+    canvas.reset()
+    message_id = str(uuid4())
+    
+    # Handle new session creation
+    if not session_id:
+        query = canvas.get_preset_param()
+        if query:
+            for ele in query:
+                if not ele["optional"]:
+                    if not kwargs.get(ele["key"]):
+                        yield get_data_openai(
+                            id=None,
+                            model=agent_id,
+                            content=f"`{ele['key']}` is required",
+                            completion_tokens=len(tiktokenenc.encode(f"`{ele['key']}` is required")),
+                            prompt_tokens=len(tiktokenenc.encode(question if question else ""))
+                        )
+                        return
+                    ele["value"] = kwargs[ele["key"]]
+                if ele["optional"]:
+                    if kwargs.get(ele["key"]):
+                        ele["value"] = kwargs[ele['key']]
+                    else:
+                        if "value" in ele:
+                            ele.pop("value")
+        
+        cvs.dsl = json.loads(str(canvas))
+        session_id = get_uuid()
+        conv = {
+            "id": session_id,
+            "dialog_id": cvs.id,
+            "user_id": kwargs.get("user_id", "") if isinstance(kwargs, dict) else "",
+            "message": [{"role": "assistant", "content": canvas.get_prologue(), "created_at": time.time()}],
+            "source": "agent",
+            "dsl": cvs.dsl
+        }
+        canvas.messages.append({"role": "user", "content": question, "id": message_id})
+        canvas.add_user_input(question)
+
+        API4ConversationService.save(**conv)
+        conv = API4Conversation(**conv)
+        if not conv.message:
+            conv.message = []
+        conv.message.append({
+            "role": "user",
+            "content": question,
+            "id": message_id
+        })
+        
+        if not conv.reference:
+            conv.reference = []
+        conv.reference.append({"chunks": [], "doc_aggs": []})
+            
+    # Handle existing session
+    else:
+        e, conv = API4ConversationService.get_by_id(session_id)
+        if not e:
+            yield get_data_openai(
+                id=session_id,
+                model=agent_id,
+                content="**ERROR**: Session not found!"
+            )
+            return
+        
+        canvas = Canvas(json.dumps(conv.dsl), tenant_id)
+        canvas.messages.append({"role": "user", "content": question, "id": message_id})
+        canvas.add_user_input(question)
+        
+        if not conv.message:
+            conv.message = []
+        conv.message.append({
+            "role": "user",
+            "content": question,
+            "id": message_id
+        })
+        
+        if not conv.reference:
+            conv.reference = []
+        conv.reference.append({"chunks": [], "doc_aggs": []})
+    
+    # Process request based on stream mode
+    final_ans = {"reference": [], "content": ""}
+    prompt_tokens = len(tiktokenenc.encode(str(question)))
+    
+    if stream:
+        try:
+            completion_tokens = 0
+            for ans in canvas.run(stream=True, bypass_begin=True):
+                if ans.get("running_status"):
+                    completion_tokens += len(tiktokenenc.encode(ans.get("content", "")))
+                    yield "data: " + json.dumps(
+                        get_data_openai(
+                            id=session_id,
+                            model=agent_id,
+                            content=ans["content"],
+                            object="chat.completion.chunk",
+                            completion_tokens=completion_tokens,
+                            prompt_tokens=prompt_tokens
+                        ),
+                        ensure_ascii=False
+                    ) + "\n\n"
+                    continue
+                
+                for k in ans.keys():
+                    final_ans[k] = ans[k]
+                
+                completion_tokens += len(tiktokenenc.encode(final_ans.get("content", "")))
+                yield "data: " + json.dumps(
+                    get_data_openai(
+                        id=session_id,
+                        model=agent_id,
+                        content=final_ans["content"],
+                        object="chat.completion.chunk",
+                        finish_reason="stop",
+                        completion_tokens=completion_tokens,
+                        prompt_tokens=prompt_tokens
+                    ),
+                    ensure_ascii=False
+                ) + "\n\n"
+            
+            # Update conversation
+            canvas.messages.append({"role": "assistant", "content": final_ans["content"], "created_at": time.time(), "id": message_id})
+            canvas.history.append(("assistant", final_ans["content"]))
+            if final_ans.get("reference"):
+                canvas.reference.append(final_ans["reference"])
+            conv.dsl = json.loads(str(canvas))
+            API4ConversationService.append_message(conv.id, conv.to_dict())
+            
+            yield "data: [DONE]\n\n"
+            
+        except Exception as e:
+            traceback.print_exc()
+            conv.dsl = json.loads(str(canvas))
+            API4ConversationService.append_message(conv.id, conv.to_dict())
+            yield "data: " + json.dumps(
+                get_data_openai(
+                    id=session_id,
+                    model=agent_id,
+                    content="**ERROR**: " + str(e),
+                    finish_reason="stop",
+                    completion_tokens=len(tiktokenenc.encode("**ERROR**: " + str(e))),
+                    prompt_tokens=prompt_tokens
+                ),
+                ensure_ascii=False
+            ) + "\n\n"
+            yield "data: [DONE]\n\n"
+    
+    else:  # Non-streaming mode
+        try:
+            all_answer_content = ""
+            for answer in canvas.run(stream=False, bypass_begin=True):
+                if answer.get("running_status"):
+                    continue
+                
+                final_ans["content"] = "\n".join(answer["content"]) if "content" in answer else ""
+                final_ans["reference"] = answer.get("reference", [])
+                all_answer_content += final_ans["content"]
+            
+            final_ans["content"] = all_answer_content
+            
+            # Update conversation
+            canvas.messages.append({"role": "assistant", "content": final_ans["content"], "created_at": time.time(), "id": message_id})
+            canvas.history.append(("assistant", final_ans["content"]))
+            if final_ans.get("reference"):
+                canvas.reference.append(final_ans["reference"])
+            conv.dsl = json.loads(str(canvas))
+            API4ConversationService.append_message(conv.id, conv.to_dict())
+            
+            # Return the response in OpenAI format
+            yield get_data_openai(
+                id=session_id,
+                model=agent_id,
+                content=final_ans["content"],
+                finish_reason="stop",
+                completion_tokens=len(tiktokenenc.encode(final_ans["content"])),
+                prompt_tokens=prompt_tokens,
+                param=canvas.get_preset_param()  # Added param info like in completion
+            )
+            
+        except Exception as e:
+            traceback.print_exc()
+            conv.dsl = json.loads(str(canvas))
+            API4ConversationService.append_message(conv.id, conv.to_dict())
+            yield get_data_openai(
+                id=session_id,
+                model=agent_id,
+                content="**ERROR**: " + str(e),
+                finish_reason="stop",
+                completion_tokens=len(tiktokenenc.encode("**ERROR**: " + str(e))),
+                prompt_tokens=prompt_tokens
+            )
+

api/db/services/common_service.py

@@ -18,21 +18,60 @@ from datetime import datetime
 import peewee
 
 from api.db.db_models import DB
-from api.utils import datetime_format, current_timestamp, get_uuid
+from api.utils import current_timestamp, datetime_format, get_uuid
 
 
 class CommonService:
+    """Base service class that provides common database operations.
+
+    This class serves as a foundation for all service classes in the application,
+    implementing standard CRUD operations and common database query patterns.
+    It uses the Peewee ORM for database interactions and provides a consistent
+    interface for database operations across all derived service classes.
+
+    Attributes:
+        model: The Peewee model class that this service operates on. Must be set by subclasses.
+    """
+
     model = None
 
     @classmethod
     @DB.connection_context()
     def query(cls, cols=None, reverse=None, order_by=None, **kwargs):
-        return cls.model.query(cols=cols, reverse=reverse,
-                               order_by=order_by, **kwargs)
+        """Execute a database query with optional column selection and ordering.
+
+        This method provides a flexible way to query the database with various filters
+        and sorting options. It supports column selection, sort order control, and
+        additional filter conditions.
+
+        Args:
+            cols (list, optional): List of column names to select. If None, selects all columns.
+            reverse (bool, optional): If True, sorts in descending order. If False, sorts in ascending order.
+            order_by (str, optional): Column name to sort results by.
+            **kwargs: Additional filter conditions passed as keyword arguments.
+
+        Returns:
+            peewee.ModelSelect: A query result containing matching records.
+        """
+        return cls.model.query(cols=cols, reverse=reverse, order_by=order_by, **kwargs)
 
     @classmethod
     @DB.connection_context()
     def get_all(cls, cols=None, reverse=None, order_by=None):
+        """Retrieve all records from the database with optional column selection and ordering.
+
+        This method fetches all records from the model's table with support for
+        column selection and result ordering. If no order_by is specified and reverse
+        is True, it defaults to ordering by create_time.
+
+        Args:
+            cols (list, optional): List of column names to select. If None, selects all columns.
+            reverse (bool, optional): If True, sorts in descending order. If False, sorts in ascending order.
+            order_by (str, optional): Column name to sort results by. Defaults to 'create_time' if reverse is specified.
+
+        Returns:
+            peewee.ModelSelect: A query containing all matching records.
+        """
         if cols:
             query_records = cls.model.select(*cols)
         else:
@@ -41,21 +80,44 @@ class CommonService:
             if not order_by or not hasattr(cls, order_by):
                 order_by = "create_time"
             if reverse is True:
-                query_records = query_records.order_by(
-                    cls.model.getter_by(order_by).desc())
+                query_records = query_records.order_by(cls.model.getter_by(order_by).desc())
             elif reverse is False:
-                query_records = query_records.order_by(
-                    cls.model.getter_by(order_by).asc())
+                query_records = query_records.order_by(cls.model.getter_by(order_by).asc())
         return query_records
 
     @classmethod
     @DB.connection_context()
     def get(cls, **kwargs):
+        """Get a single record matching the given criteria.
+
+        This method retrieves a single record from the database that matches
+        the specified filter conditions.
+
+        Args:
+            **kwargs: Filter conditions as keyword arguments.
+
+        Returns:
+            Model instance: Single matching record.
+
+        Raises:
+            peewee.DoesNotExist: If no matching record is found.
+        """
         return cls.model.get(**kwargs)
 
     @classmethod
     @DB.connection_context()
     def get_or_none(cls, **kwargs):
+        """Get a single record or None if not found.
+
+        This method attempts to retrieve a single record matching the given criteria,
+        returning None if no match is found instead of raising an exception.
+
+        Args:
+            **kwargs: Filter conditions as keyword arguments.
+
+        Returns:
+            Model instance or None: Matching record if found, None otherwise.
+        """
         try:
             return cls.model.get(**kwargs)
         except peewee.DoesNotExist:
@@ -64,14 +126,34 @@ class CommonService:
     @classmethod
     @DB.connection_context()
     def save(cls, **kwargs):
-        # if "id" not in kwargs:
-        #    kwargs["id"] = get_uuid()
+        """Save a new record to database.
+
+        This method creates a new record in the database with the provided field values,
+        forcing an insert operation rather than an update.
+
+        Args:
+            **kwargs: Record field values as keyword arguments.
+
+        Returns:
+            Model instance: The created record object.
+        """
         sample_obj = cls.model(**kwargs).save(force_insert=True)
         return sample_obj
 
     @classmethod
     @DB.connection_context()
     def insert(cls, **kwargs):
+        """Insert a new record with automatic ID and timestamps.
+
+        This method creates a new record with automatically generated ID and timestamp fields.
+        It handles the creation of create_time, create_date, update_time, and update_date fields.
+
+        Args:
+            **kwargs: Record field values as keyword arguments.
+
+        Returns:
+            Model instance: The newly created record object.
+        """
         if "id" not in kwargs:
             kwargs["id"] = get_uuid()
         kwargs["create_time"] = current_timestamp()
@@ -84,26 +166,49 @@ class CommonService:
     @classmethod
     @DB.connection_context()
     def insert_many(cls, data_list, batch_size=100):
+        """Insert multiple records in batches.
+
+        This method efficiently inserts multiple records into the database using batch processing.
+        It automatically sets creation timestamps for all records.
+
+        Args:
+            data_list (list): List of dictionaries containing record data to insert.
+            batch_size (int, optional): Number of records to insert in each batch. Defaults to 100.
+        """
         with DB.atomic():
             for d in data_list:
                 d["create_time"] = current_timestamp()
                 d["create_date"] = datetime_format(datetime.now())
             for i in range(0, len(data_list), batch_size):
-                cls.model.insert_many(data_list[i:i + batch_size]).execute()
+                cls.model.insert_many(data_list[i : i + batch_size]).execute()
 
     @classmethod
     @DB.connection_context()
     def update_many_by_id(cls, data_list):
+        """Update multiple records by their IDs.
+
+        This method updates multiple records in the database, identified by their IDs.
+        It automatically updates the update_time and update_date fields for each record.
+
+        Args:
+            data_list (list): List of dictionaries containing record data to update.
+                             Each dictionary must include an 'id' field.
+        """
         with DB.atomic():
             for data in data_list:
                 data["update_time"] = current_timestamp()
                 data["update_date"] = datetime_format(datetime.now())
-                cls.model.update(data).where(
-                    cls.model.id == data["id"]).execute()
+                cls.model.update(data).where(cls.model.id == data["id"]).execute()
 
     @classmethod
     @DB.connection_context()
     def update_by_id(cls, pid, data):
+        # Update a single record by ID
+        # Args:
+        #     pid: Record ID
+        #     data: Updated field values
+        # Returns:
+        #     Number of records updated
         data["update_time"] = current_timestamp()
         data["update_date"] = datetime_format(datetime.now())
         num = cls.model.update(data).where(cls.model.id == pid).execute()
@@ -112,15 +217,28 @@ class CommonService:
     @classmethod
     @DB.connection_context()
     def get_by_id(cls, pid):
+        # Get a record by ID
+        # Args:
+        #     pid: Record ID
+        # Returns:
+        #     Tuple of (success, record)
         try:
-            obj = cls.model.query(id=pid)[0]
-            return True, obj
+            obj = cls.model.get_or_none(cls.model.id == pid)
+            if obj:
+                return True, obj
         except Exception:
-            return False, None
+            pass
+        return False, None
 
     @classmethod
     @DB.connection_context()
     def get_by_ids(cls, pids, cols=None):
+        # Get multiple records by their IDs
+        # Args:
+        #     pids: List of record IDs
+        #     cols: List of columns to select
+        # Returns:
+        #     Query of matching records
         if cols:
             objs = cls.model.select(*cols)
         else:
@@ -130,11 +248,33 @@ class CommonService:
     @classmethod
     @DB.connection_context()
     def delete_by_id(cls, pid):
+        # Delete a record by ID
+        # Args:
+        #     pid: Record ID
+        # Returns:
+        #     Number of records deleted
         return cls.model.delete().where(cls.model.id == pid).execute()
+    
+    @classmethod
+    @DB.connection_context()
+    def delete_by_ids(cls, pids):
+        # Delete multiple records by their IDs
+        # Args:
+        #     pids: List of record IDs
+        # Returns:
+        #     Number of records deleted
+        with DB.atomic():
+            res = cls.model.delete().where(cls.model.id.in_(pids)).execute()
+            return res
 
     @classmethod
     @DB.connection_context()
     def filter_delete(cls, filters):
+        # Delete records matching given filters
+        # Args:
+        #     filters: List of filter conditions
+        # Returns:
+        #     Number of records deleted
         with DB.atomic():
             num = cls.model.delete().where(*filters).execute()
             return num
@@ -142,42 +282,51 @@ class CommonService:
     @classmethod
     @DB.connection_context()
     def filter_update(cls, filters, update_data):
+        # Update records matching given filters
+        # Args:
+        #     filters: List of filter conditions
+        #     update_data: Updated field values
+        # Returns:
+        #     Number of records updated
         with DB.atomic():
             return cls.model.update(update_data).where(*filters).execute()
 
     @staticmethod
     def cut_list(tar_list, n):
+        # Split a list into chunks of size n
+        # Args:
+        #     tar_list: List to split
+        #     n: Chunk size
+        # Returns:
+        #     List of tuples containing chunks
         length = len(tar_list)
         arr = range(length)
-        result = [tuple(tar_list[x:(x + n)]) for x in arr[::n]]
+        result = [tuple(tar_list[x : (x + n)]) for x in arr[::n]]
         return result
 
     @classmethod
     @DB.connection_context()
-    def filter_scope_list(cls, in_key, in_filters_list,
-                          filters=None, cols=None):
+    def filter_scope_list(cls, in_key, in_filters_list, filters=None, cols=None):
+        # Get records matching IN clause filters with optional column selection
+        # Args:
+        #     in_key: Field name for IN clause
+        #     in_filters_list: List of values for IN clause
+        #     filters: Additional filter conditions
+        #     cols: List of columns to select
+        # Returns:
+        #     List of matching records
         in_filters_tuple_list = cls.cut_list(in_filters_list, 20)
         if not filters:
             filters = []
         res_list = []
         if cols:
             for i in in_filters_tuple_list:
-                query_records = cls.model.select(
-                    *
-                    cols).where(
-                    getattr(
-                        cls.model,
-                        in_key).in_(i),
-                    *
-                    filters)
+                query_records = cls.model.select(*cols).where(getattr(cls.model, in_key).in_(i), *filters)
                 if query_records:
-                    res_list.extend(
-                        [query_record for query_record in query_records])
+                    res_list.extend([query_record for query_record in query_records])
         else:
             for i in in_filters_tuple_list:
-                query_records = cls.model.select().where(
-                    getattr(cls.model, in_key).in_(i), *filters)
+                query_records = cls.model.select().where(getattr(cls.model, in_key).in_(i), *filters)
                 if query_records:
-                    res_list.extend(
-                        [query_record for query_record in query_records])
+                    res_list.extend([query_record for query_record in query_records])
         return res_list

api/db/services/conversation_service.py

@@ -23,6 +23,8 @@ from api.db.services.dialog_service import DialogService, chat
 from api.utils import get_uuid
 import json
 
+from rag.prompts import chunks_format
+
 
 class ConversationService(CommonService):
     model = Conversation
@@ -53,18 +55,7 @@ def structure_answer(conv, ans, message_id, session_id):
         reference = {}
         ans["reference"] = {}
 
-    def get_value(d, k1, k2):
-        return d.get(k1, d.get(k2))
-
-    chunk_list = [{
-        "id": get_value(chunk, "chunk_id", "id"),
-        "content": get_value(chunk, "content", "content_with_weight"),
-        "document_id": get_value(chunk, "doc_id", "document_id"),
-        "document_name": get_value(chunk, "docnm_kwd", "document_name"),
-        "dataset_id": get_value(chunk, "kb_id", "dataset_id"),
-        "image_id": get_value(chunk, "image_id", "img_id"),
-        "positions": get_value(chunk, "positions", "position_int"),
-    } for chunk in reference.get("chunks", [])]
+    chunk_list = chunks_format(reference)
 
     reference["chunks"] = chunk_list
     ans["id"] = message_id
@@ -99,17 +90,18 @@ def completion(tenant_id, chat_id, question, name="New session", session_id=None
             "user_id": kwargs.get("user_id", "")
         }
         ConversationService.save(**conv)
-        yield "data:" + json.dumps({"code": 0, "message": "",
-                                    "data": {
-                                        "answer": conv["message"][0]["content"],
-                                        "reference": {},
-                                        "audio_binary": None,
-                                        "id": None,
-                                        "session_id": session_id
-                                    }},
-                                   ensure_ascii=False) + "\n\n"
-        yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
-        return
+        if stream:
+            yield "data:" + json.dumps({"code": 0, "message": "",
+                                        "data": {
+                                            "answer": conv["message"][0]["content"],
+                                            "reference": {},
+                                            "audio_binary": None,
+                                            "id": None,
+                                            "session_id": session_id
+                                        }},
+                                    ensure_ascii=False) + "\n\n"
+            yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
+            return
 
     conv = ConversationService.query(id=session_id, dialog_id=chat_id)
     if not conv:
@@ -132,6 +124,8 @@ def completion(tenant_id, chat_id, question, name="New session", session_id=None
     message_id = msg[-1].get("id")
     e, dia = DialogService.get_by_id(conv.dialog_id)
 
+    kb_ids = kwargs.get("kb_ids",[])
+    dia.kb_ids = list(set(dia.kb_ids + kb_ids))
     if not conv.reference:
         conv.reference = []
     conv.message.append({"role": "assistant", "content": "", "id": message_id})

api/db/services/document_service.py

@@ -13,9 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import logging
-import xxhash
 import json
+import logging
 import random
 import re
 from concurrent.futures import ThreadPoolExecutor
@@ -23,23 +22,23 @@ from copy import deepcopy
 from datetime import datetime
 from io import BytesIO
 
+import trio
+import xxhash
 from peewee import fn
 
-from api.db.db_utils import bulk_insert_into_db
 from api import settings
-from api.utils import current_timestamp, get_format_time, get_uuid
-from graphrag.general.mind_map_extractor import MindMapExtractor
-from rag.settings import SVR_QUEUE_NAME
-from rag.utils.storage_factory import STORAGE_IMPL
-from rag.nlp import search, rag_tokenizer
-
-from api.db import FileType, TaskStatus, ParserType, LLMType
-from api.db.db_models import DB, Knowledgebase, Tenant, Task, UserTenant
-from api.db.db_models import Document
+from api.constants import IMG_BASE64_PREFIX
+from api.db import FileType, LLMType, ParserType, StatusEnum, TaskStatus, UserTenantRole
+from api.db.db_models import DB, Document, Knowledgebase, Task, Tenant, UserTenant
+from api.db.db_utils import bulk_insert_into_db
 from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db import StatusEnum
+from api.utils import current_timestamp, get_format_time, get_uuid
+from rag.nlp import rag_tokenizer, search
+from rag.settings import get_svr_queue_name, SVR_CONSUMER_GROUP_NAME
 from rag.utils.redis_conn import REDIS_CONN
+from rag.utils.storage_factory import STORAGE_IMPL
+from rag.utils.doc_store_conn import OrderByExpr
 
 
 class DocumentService(CommonService):
@@ -73,7 +72,7 @@ class DocumentService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_by_kb_id(cls, kb_id, page_number, items_per_page,
-                     orderby, desc, keywords):
+                     orderby, desc, keywords, run_status, types):
         if keywords:
             docs = cls.model.select().where(
                 (cls.model.kb_id == kb_id),
@@ -81,24 +80,66 @@ class DocumentService(CommonService):
             )
         else:
             docs = cls.model.select().where(cls.model.kb_id == kb_id)
+
+        if run_status:
+            docs = docs.where(cls.model.run.in_(run_status))
+        if types:
+            docs = docs.where(cls.model.type.in_(types))
+
         count = docs.count()
         if desc:
             docs = docs.order_by(cls.model.getter_by(orderby).desc())
         else:
             docs = docs.order_by(cls.model.getter_by(orderby).asc())
 
-        docs = docs.paginate(page_number, items_per_page)
+
+        if page_number and items_per_page:
+            docs = docs.paginate(page_number, items_per_page)
 
         return list(docs.dicts()), count
 
+    @classmethod
+    @DB.connection_context()
+    def count_by_kb_id(cls, kb_id, keywords, run_status, types):
+        if keywords:
+            docs = cls.model.select().where(
+                (cls.model.kb_id == kb_id),
+                (fn.LOWER(cls.model.name).contains(keywords.lower()))
+            )
+        else:
+            docs = cls.model.select().where(cls.model.kb_id == kb_id)
+
+        if run_status:
+            docs = docs.where(cls.model.run.in_(run_status))
+        if types:
+            docs = docs.where(cls.model.type.in_(types))
+
+        count = docs.count()
+
+        return count
+
+    @classmethod
+    @DB.connection_context()
+    def get_total_size_by_kb_id(cls, kb_id, keywords="", run_status=[], types=[]):
+        query = cls.model.select(fn.COALESCE(fn.SUM(cls.model.size), 0)).where(
+            cls.model.kb_id == kb_id
+        )
+
+        if keywords:
+            query = query.where(fn.LOWER(cls.model.name).contains(keywords.lower()))
+        if run_status:
+            query = query.where(cls.model.run.in_(run_status))
+        if types:
+            query = query.where(cls.model.type.in_(types))
+
+        return int(query.scalar()) or 0
+
     @classmethod
     @DB.connection_context()
     def insert(cls, doc):
         if not cls.save(**doc):
             raise RuntimeError("Database error (Document)!")
-        e, kb = KnowledgebaseService.get_by_id(doc["kb_id"])
-        if not KnowledgebaseService.update_by_id(
-                kb.id, {"doc_num": kb.doc_num + 1}):
+        if not KnowledgebaseService.atomic_increase_doc_num_by_id(doc["kb_id"]):
             raise RuntimeError("Database error (Knowledgebase)!")
         return Document(**doc)
 
@@ -107,15 +148,38 @@ class DocumentService(CommonService):
     def remove_document(cls, doc, tenant_id):
         cls.clear_chunk_num(doc.id)
         try:
+            page = 0
+            page_size = 1000
+            all_chunk_ids = []
+            while True:
+                chunks = settings.docStoreConn.search(["img_id"], [], {"doc_id": doc.id}, [], OrderByExpr(),
+                                                      page * page_size, page_size, search.index_name(tenant_id),
+                                                      [doc.kb_id])
+                chunk_ids = settings.docStoreConn.getChunkIds(chunks)
+                if not chunk_ids:
+                    break
+                all_chunk_ids.extend(chunk_ids)
+                page += 1
+            for cid in all_chunk_ids:
+                if STORAGE_IMPL.obj_exist(doc.kb_id, cid):
+                    STORAGE_IMPL.rm(doc.kb_id, cid)
+            if doc.thumbnail and not doc.thumbnail.startswith(IMG_BASE64_PREFIX):
+                if STORAGE_IMPL.obj_exist(doc.kb_id, doc.thumbnail):
+                    STORAGE_IMPL.rm(doc.kb_id, doc.thumbnail)
             settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id)
-            settings.docStoreConn.update({"kb_id": doc.kb_id, "knowledge_graph_kwd": ["entity", "relation", "graph", "community_report"], "source_id": doc.id},
-                                         {"remove": {"source_id": doc.id}},
-                                         search.index_name(tenant_id), doc.kb_id)
-            settings.docStoreConn.update({"kb_id": doc.kb_id, "knowledge_graph_kwd": ["graph"]},
-                                         {"removed_kwd": "Y"},
-                                         search.index_name(tenant_id), doc.kb_id)
-            settings.docStoreConn.delete({"kb_id": doc.kb_id, "knowledge_graph_kwd": ["entity", "relation", "graph", "community_report"], "must_not": {"exists": "source_id"}},
-                                         search.index_name(tenant_id), doc.kb_id)
+
+            graph_source = settings.docStoreConn.getFields(
+                settings.docStoreConn.search(["source_id"], [], {"kb_id": doc.kb_id, "knowledge_graph_kwd": ["graph"]}, [], OrderByExpr(), 0, 1, search.index_name(tenant_id), [doc.kb_id]), ["source_id"]
+            )
+            if len(graph_source) > 0 and doc.id in list(graph_source.values())[0]["source_id"]:
+                settings.docStoreConn.update({"kb_id": doc.kb_id, "knowledge_graph_kwd": ["entity", "relation", "graph", "subgraph", "community_report"], "source_id": doc.id},
+                                            {"remove": {"source_id": doc.id}},
+                                            search.index_name(tenant_id), doc.kb_id)
+                settings.docStoreConn.update({"kb_id": doc.kb_id, "knowledge_graph_kwd": ["graph"]},
+                                            {"removed_kwd": "Y"},
+                                            search.index_name(tenant_id), doc.kb_id)
+                settings.docStoreConn.delete({"kb_id": doc.kb_id, "knowledge_graph_kwd": ["entity", "relation", "graph", "subgraph", "community_report"], "must_not": {"exists": "source_id"}},
+                                            search.index_name(tenant_id), doc.kb_id)
         except Exception:
             pass
         return cls.delete_by_id(doc.id)
@@ -174,9 +238,9 @@ class DocumentService(CommonService):
                 "Document not found which is supposed to be there")
         num = Knowledgebase.update(
             token_num=Knowledgebase.token_num +
-                      token_num,
+            token_num,
             chunk_num=Knowledgebase.chunk_num +
-                      chunk_num).where(
+            chunk_num).where(
             Knowledgebase.id == kb_id).execute()
         return num
 
@@ -192,9 +256,9 @@ class DocumentService(CommonService):
                 "Document not found which is supposed to be there")
         num = Knowledgebase.update(
             token_num=Knowledgebase.token_num -
-                      token_num,
+            token_num,
             chunk_num=Knowledgebase.chunk_num -
-                      chunk_num
+            chunk_num
         ).where(
             Knowledgebase.id == kb_id).execute()
         return num
@@ -207,9 +271,9 @@ class DocumentService(CommonService):
 
         num = Knowledgebase.update(
             token_num=Knowledgebase.token_num -
-                      doc.token_num,
+            doc.token_num,
             chunk_num=Knowledgebase.chunk_num -
-                      doc.chunk_num,
+            doc.chunk_num,
             doc_num=Knowledgebase.doc_num - 1
         ).where(
             Knowledgebase.id == doc.kb_id).execute()
@@ -221,7 +285,7 @@ class DocumentService(CommonService):
         docs = cls.model.select(
             Knowledgebase.tenant_id).join(
             Knowledgebase, on=(
-                    Knowledgebase.id == cls.model.kb_id)).where(
+                Knowledgebase.id == cls.model.kb_id)).where(
             cls.model.id == doc_id, Knowledgebase.status == StatusEnum.VALID.value)
         docs = docs.dicts()
         if not docs:
@@ -243,7 +307,7 @@ class DocumentService(CommonService):
         docs = cls.model.select(
             Knowledgebase.tenant_id).join(
             Knowledgebase, on=(
-                    Knowledgebase.id == cls.model.kb_id)).where(
+                Knowledgebase.id == cls.model.kb_id)).where(
             cls.model.name == name, Knowledgebase.status == StatusEnum.VALID.value)
         docs = docs.dicts()
         if not docs:
@@ -256,7 +320,7 @@ class DocumentService(CommonService):
         docs = cls.model.select(
             cls.model.id).join(
             Knowledgebase, on=(
-                    Knowledgebase.id == cls.model.kb_id)
+                Knowledgebase.id == cls.model.kb_id)
         ).join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
                ).where(cls.model.id == doc_id, UserTenant.user_id == user_id).paginate(0, 1)
         docs = docs.dicts()
@@ -267,11 +331,18 @@ class DocumentService(CommonService):
     @classmethod
     @DB.connection_context()
     def accessible4deletion(cls, doc_id, user_id):
-        docs = cls.model.select(
-            cls.model.id).join(
+        docs = cls.model.select(cls.model.id
+        ).join(
             Knowledgebase, on=(
-                    Knowledgebase.id == cls.model.kb_id)
-        ).where(cls.model.id == doc_id, Knowledgebase.created_by == user_id).paginate(0, 1)
+                Knowledgebase.id == cls.model.kb_id)
+        ).join(
+            UserTenant, on=(
+                (UserTenant.tenant_id == Knowledgebase.created_by) & (UserTenant.user_id == user_id))
+        ).where(
+            cls.model.id == doc_id,
+            UserTenant.status == StatusEnum.VALID.value,
+            ((UserTenant.role == UserTenantRole.NORMAL) | (UserTenant.role == UserTenantRole.OWNER))
+        ).paginate(0, 1)
         docs = docs.dicts()
         if not docs:
             return False
@@ -283,7 +354,7 @@ class DocumentService(CommonService):
         docs = cls.model.select(
             Knowledgebase.embd_id).join(
             Knowledgebase, on=(
-                    Knowledgebase.id == cls.model.kb_id)).where(
+                Knowledgebase.id == cls.model.kb_id)).where(
             cls.model.id == doc_id, Knowledgebase.status == StatusEnum.VALID.value)
         docs = docs.dicts()
         if not docs:
@@ -306,9 +377,9 @@ class DocumentService(CommonService):
                 Tenant.asr_id,
                 Tenant.llm_id,
             )
-                .join(Knowledgebase, on=(cls.model.kb_id == Knowledgebase.id))
-                .join(Tenant, on=(Knowledgebase.tenant_id == Tenant.id))
-                .where(cls.model.id == doc_id)
+            .join(Knowledgebase, on=(cls.model.kb_id == Knowledgebase.id))
+            .join(Tenant, on=(Knowledgebase.tenant_id == Tenant.id))
+            .where(cls.model.id == doc_id)
         )
         configs = configs.dicts()
         if not configs:
@@ -325,6 +396,15 @@ class DocumentService(CommonService):
         if not doc_id:
             return
         return doc_id[0]["id"]
+    
+    @classmethod
+    @DB.connection_context()
+    def get_doc_ids_by_doc_names(cls, doc_names):
+        if not doc_names:
+            return []
+
+        query = cls.model.select(cls.model.id).where(cls.model.name.in_(doc_names))
+        return list(query.scalars().iterator())
 
     @classmethod
     @DB.connection_context()
@@ -336,6 +416,8 @@ class DocumentService(CommonService):
     @classmethod
     @DB.connection_context()
     def update_parser_config(cls, id, config):
+        if not config:
+            return
         e, d = cls.get_by_id(id)
         if not e:
             raise LookupError(f"Document({id}) not found.")
@@ -373,15 +455,14 @@ class DocumentService(CommonService):
                     "process_begin_at": get_format_time()
                     })
 
+    @classmethod
+    @DB.connection_context()
+    def update_meta_fields(cls, doc_id, meta_fields):
+        return cls.update_by_id(doc_id, {"meta_fields": meta_fields})
+
     @classmethod
     @DB.connection_context()
     def update_progress(cls):
-        MSG = {
-            "raptor": "Start RAPTOR (Recursive Abstractive Processing for Tree-Organized Retrieval).",
-            "graphrag": "Start Graph Extraction",
-            "graph_resolution": "Start Graph Resolution",
-            "graph_community": "Start Graph Community Reports Generation"
-        }
         docs = cls.get_unfinished_docs()
         for d in docs:
             try:
@@ -392,37 +473,34 @@ class DocumentService(CommonService):
                 prg = 0
                 finished = True
                 bad = 0
+                has_raptor = False
+                has_graphrag = False
                 e, doc = DocumentService.get_by_id(d["id"])
                 status = doc.run  # TaskStatus.RUNNING.value
+                priority = 0
                 for t in tsks:
                     if 0 <= t.progress < 1:
                         finished = False
-                    prg += t.progress if t.progress >= 0 else 0
-                    if t.progress_msg not in msg:
-                        msg.append(t.progress_msg)
                     if t.progress == -1:
                         bad += 1
+                    prg += t.progress if t.progress >= 0 else 0
+                    if t.progress_msg.strip():
+                        msg.append(t.progress_msg)
+                    if t.task_type == "raptor":
+                        has_raptor = True
+                    elif t.task_type == "graphrag":
+                        has_graphrag = True
+                    priority = max(priority, t.priority)
                 prg /= len(tsks)
                 if finished and bad:
                     prg = -1
                     status = TaskStatus.FAIL.value
                 elif finished:
-                    m = "\n".join(sorted(msg))
-                    if d["parser_config"].get("raptor", {}).get("use_raptor") and m.find(MSG["raptor"]) < 0:
-                        queue_raptor_o_graphrag_tasks(d, "raptor", MSG["raptor"])
+                    if d["parser_config"].get("raptor", {}).get("use_raptor") and not has_raptor:
+                        queue_raptor_o_graphrag_tasks(d, "raptor", priority)
                         prg = 0.98 * len(tsks) / (len(tsks) + 1)
-                    elif d["parser_config"].get("graphrag", {}).get("use_graphrag") and m.find(MSG["graphrag"]) < 0:
-                        queue_raptor_o_graphrag_tasks(d, "graphrag", MSG["graphrag"])
-                        prg = 0.98 * len(tsks) / (len(tsks) + 1)
-                    elif d["parser_config"].get("graphrag", {}).get("use_graphrag") \
-                        and d["parser_config"].get("graphrag", {}).get("resolution") \
-                        and m.find(MSG["graph_resolution"]) < 0:
-                        queue_raptor_o_graphrag_tasks(d, "graph_resolution", MSG["graph_resolution"])
-                        prg = 0.98 * len(tsks) / (len(tsks) + 1)
-                    elif d["parser_config"].get("graphrag", {}).get("use_graphrag") \
-                        and d["parser_config"].get("graphrag", {}).get("community") \
-                        and m.find(MSG["graph_community"]) < 0:
-                        queue_raptor_o_graphrag_tasks(d, "graph_community", MSG["graph_community"])
+                    elif d["parser_config"].get("graphrag", {}).get("use_graphrag") and not has_graphrag:
+                        queue_raptor_o_graphrag_tasks(d, "graphrag", priority)
                         prg = 0.98 * len(tsks) / (len(tsks) + 1)
                     else:
                         status = TaskStatus.DONE.value
@@ -431,12 +509,14 @@ class DocumentService(CommonService):
                 info = {
                     "process_duation": datetime.timestamp(
                         datetime.now()) -
-                                       d["process_begin_at"].timestamp(),
+                    d["process_begin_at"].timestamp(),
                     "run": status}
                 if prg != 0:
                     info["progress"] = prg
                 if msg:
                     info["progress_msg"] = msg
+                else:
+                    info["progress_msg"] = "%d tasks are ahead in the queue..."%get_queue_length(priority)
                 cls.update_by_id(d["id"], info)
             except Exception as e:
                 if str(e).find("'0'") < 0:
@@ -459,7 +539,7 @@ class DocumentService(CommonService):
         return False
 
 
-def queue_raptor_o_graphrag_tasks(doc, ty, msg):
+def queue_raptor_o_graphrag_tasks(doc, ty, priority):
     chunking_config = DocumentService.get_chunking_config(doc["id"])
     hasher = xxhash.xxh64()
     for field in sorted(chunking_config.keys()):
@@ -472,7 +552,8 @@ def queue_raptor_o_graphrag_tasks(doc, ty, msg):
             "doc_id": doc["id"],
             "from_page": 100000000,
             "to_page": 100000000,
-            "progress_msg":  datetime.now().strftime("%H:%M:%S") + " " + msg
+            "task_type": ty,
+            "progress_msg":  datetime.now().strftime("%H:%M:%S") + " created task " + ty
         }
 
     task = new_task()
@@ -481,18 +562,22 @@ def queue_raptor_o_graphrag_tasks(doc, ty, msg):
     hasher.update(ty.encode("utf-8"))
     task["digest"] = hasher.hexdigest()
     bulk_insert_into_db(Task, [task], True)
-    task["task_type"] = ty
-    assert REDIS_CONN.queue_product(SVR_QUEUE_NAME, message=task), "Can't access Redis. Please check the Redis' status."
+    assert REDIS_CONN.queue_product(get_svr_queue_name(priority), message=task), "Can't access Redis. Please check the Redis' status."
+
+
+def get_queue_length(priority):
+    group_info = REDIS_CONN.queue_info(get_svr_queue_name(priority), SVR_CONSUMER_GROUP_NAME)
+    return int(group_info.get("lag", 0))
 
 
 def doc_upload_and_parse(conversation_id, file_objs, user_id):
-    from rag.app import presentation, picture, naive, audio, email
+    from api.db.services.api_service import API4ConversationService
+    from api.db.services.conversation_service import ConversationService
     from api.db.services.dialog_service import DialogService
     from api.db.services.file_service import FileService
     from api.db.services.llm_service import LLMBundle
     from api.db.services.user_service import TenantService
-    from api.db.services.api_service import API4ConversationService
-    from api.db.services.conversation_service import ConversationService
+    from rag.app import audio, email, naive, picture, presentation
 
     e, conv = ConversationService.get_by_id(conversation_id)
     if not e:
@@ -500,6 +585,9 @@ def doc_upload_and_parse(conversation_id, file_objs, user_id):
     assert e, "Conversation not found!"
 
     e, dia = DialogService.get_by_id(conv.dialog_id)
+    if not dia.kb_ids:
+        raise LookupError("No knowledge base associated with this conversation. "
+                          "Please add a knowledge base before uploading documents")
     kb_id = dia.kb_ids[0]
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
@@ -588,10 +676,11 @@ def doc_upload_and_parse(conversation_id, file_objs, user_id):
         cks = [c for c in docs if c["doc_id"] == doc_id]
 
         if parser_ids[doc_id] != ParserType.PICTURE.value:
+            from graphrag.general.mind_map_extractor import MindMapExtractor
             mindmap = MindMapExtractor(llm_bdl)
             try:
-                mind_map = json.dumps(mindmap([c["content_with_weight"] for c in docs if c["doc_id"] == doc_id]).output,
-                                      ensure_ascii=False, indent=2)
+                mind_map = trio.run(mindmap, [c["content_with_weight"] for c in docs if c["doc_id"] == doc_id])
+                mind_map = json.dumps(mind_map.output, ensure_ascii=False, indent=2)
                 if len(mind_map) < 32:
                     raise Exception("Few content: " + mind_map)
                 cks.append({

api/db/services/file_service.py

@@ -14,44 +14,47 @@
 #  limitations under the License.
 #
 import logging
-import re
 import os
+import re
 from concurrent.futures import ThreadPoolExecutor
 
 from flask_login import current_user
 from peewee import fn
 
-from api.db import FileType, KNOWLEDGEBASE_FOLDER_NAME, FileSource, ParserType
-from api.db.db_models import DB, File2Document, Knowledgebase
-from api.db.db_models import File, Document
+from api.constants import FILE_NAME_LEN_LIMIT
+from api.db import KNOWLEDGEBASE_FOLDER_NAME, FileSource, FileType, ParserType
+from api.db.db_models import DB, Document, File, File2Document, Knowledgebase
 from api.db.services import duplicate_name
 from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.utils import get_uuid
-from api.utils.file_utils import filename_type, thumbnail_img
+from api.utils.file_utils import filename_type, read_potential_broken_pdf, thumbnail_img
 from rag.utils.storage_factory import STORAGE_IMPL
 
 
 class FileService(CommonService):
+    # Service class for managing file operations and storage
     model = File
 
     @classmethod
     @DB.connection_context()
-    def get_by_pf_id(cls, tenant_id, pf_id, page_number, items_per_page,
-                     orderby, desc, keywords):
+    def get_by_pf_id(cls, tenant_id, pf_id, page_number, items_per_page, orderby, desc, keywords):
+        # Get files by parent folder ID with pagination and filtering
+        # Args:
+        #     tenant_id: ID of the tenant
+        #     pf_id: Parent folder ID
+        #     page_number: Page number for pagination
+        #     items_per_page: Number of items per page
+        #     orderby: Field to order by
+        #     desc: Boolean indicating descending order
+        #     keywords: Search keywords
+        # Returns:
+        #     Tuple of (file_list, total_count)
         if keywords:
-            files = cls.model.select().where(
-                (cls.model.tenant_id == tenant_id),
-                (cls.model.parent_id == pf_id),
-                (fn.LOWER(cls.model.name).contains(keywords.lower())),
-                ~(cls.model.id == pf_id)
-            )
+            files = cls.model.select().where((cls.model.tenant_id == tenant_id), (cls.model.parent_id == pf_id), (fn.LOWER(cls.model.name).contains(keywords.lower())), ~(cls.model.id == pf_id))
         else:
-            files = cls.model.select().where((cls.model.tenant_id == tenant_id),
-                                             (cls.model.parent_id == pf_id),
-                                             ~(cls.model.id == pf_id)
-                                             )
+            files = cls.model.select().where((cls.model.tenant_id == tenant_id), (cls.model.parent_id == pf_id), ~(cls.model.id == pf_id))
         count = files.count()
         if desc:
             files = files.order_by(cls.model.getter_by(orderby).desc())
@@ -64,37 +67,54 @@ class FileService(CommonService):
         for file in res_files:
             if file["type"] == FileType.FOLDER.value:
                 file["size"] = cls.get_folder_size(file["id"])
-                file['kbs_info'] = []
-                children = list(cls.model.select().where(
-                    (cls.model.tenant_id == tenant_id),
-                    (cls.model.parent_id == file["id"]),
-                    ~(cls.model.id == file["id"]),
-                ).dicts())
-                file["has_child_folder"] = any(value["type"] == FileType.FOLDER.value for value in children)                       
+                file["kbs_info"] = []
+                children = list(
+                    cls.model.select()
+                    .where(
+                        (cls.model.tenant_id == tenant_id),
+                        (cls.model.parent_id == file["id"]),
+                        ~(cls.model.id == file["id"]),
+                    )
+                    .dicts()
+                )
+                file["has_child_folder"] = any(value["type"] == FileType.FOLDER.value for value in children)
                 continue
-            kbs_info = cls.get_kb_id_by_file_id(file['id'])
-            file['kbs_info'] = kbs_info
+            kbs_info = cls.get_kb_id_by_file_id(file["id"])
+            file["kbs_info"] = kbs_info
 
         return res_files, count
 
     @classmethod
     @DB.connection_context()
     def get_kb_id_by_file_id(cls, file_id):
-        kbs = (cls.model.select(*[Knowledgebase.id, Knowledgebase.name])
-               .join(File2Document, on=(File2Document.file_id == file_id))
-               .join(Document, on=(File2Document.document_id == Document.id))
-               .join(Knowledgebase, on=(Knowledgebase.id == Document.kb_id))
-               .where(cls.model.id == file_id))
+        # Get knowledge base IDs associated with a file
+        # Args:
+        #     file_id: File ID
+        # Returns:
+        #     List of dictionaries containing knowledge base IDs and names
+        kbs = (
+            cls.model.select(*[Knowledgebase.id, Knowledgebase.name])
+            .join(File2Document, on=(File2Document.file_id == file_id))
+            .join(Document, on=(File2Document.document_id == Document.id))
+            .join(Knowledgebase, on=(Knowledgebase.id == Document.kb_id))
+            .where(cls.model.id == file_id)
+        )
         if not kbs:
             return []
         kbs_info_list = []
         for kb in list(kbs.dicts()):
-            kbs_info_list.append({"kb_id": kb['id'], "kb_name": kb['name']})
+            kbs_info_list.append({"kb_id": kb["id"], "kb_name": kb["name"]})
         return kbs_info_list
 
     @classmethod
     @DB.connection_context()
     def get_by_pf_id_name(cls, id, name):
+        # Get file by parent folder ID and name
+        # Args:
+        #     id: Parent folder ID
+        #     name: File name
+        # Returns:
+        #     File object or None if not found
         file = cls.model.select().where((cls.model.parent_id == id) & (cls.model.name == name))
         if file.count():
             e, file = cls.get_by_id(file[0].id)
@@ -106,6 +126,14 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_id_list_by_id(cls, id, name, count, res):
+        # Recursively get list of file IDs by traversing folder structure
+        # Args:
+        #     id: Starting folder ID
+        #     name: List of folder names to traverse
+        #     count: Current depth in traversal
+        #     res: List to store results
+        # Returns:
+        #     List of file IDs
         if count < len(name):
             file = cls.get_by_pf_id_name(id, name[count])
             if file:
@@ -119,6 +147,12 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_all_innermost_file_ids(cls, folder_id, result_ids):
+        # Get IDs of all files in the deepest level of folders
+        # Args:
+        #     folder_id: Starting folder ID
+        #     result_ids: List to store results
+        # Returns:
+        #     List of file IDs
         subfolders = cls.model.select().where(cls.model.parent_id == folder_id)
         if subfolders.exists():
             for subfolder in subfolders:
@@ -130,24 +164,30 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def create_folder(cls, file, parent_id, name, count):
+        # Recursively create folder structure
+        # Args:
+        #     file: Current file object
+        #     parent_id: Parent folder ID
+        #     name: List of folder names to create
+        #     count: Current depth in creation
+        # Returns:
+        #     Created file object
         if count > len(name) - 2:
             return file
         else:
-            file = cls.insert({
-                "id": get_uuid(),
-                "parent_id": parent_id,
-                "tenant_id": current_user.id,
-                "created_by": current_user.id,
-                "name": name[count],
-                "location": "",
-                "size": 0,
-                "type": FileType.FOLDER.value
-            })
+            file = cls.insert(
+                {"id": get_uuid(), "parent_id": parent_id, "tenant_id": current_user.id, "created_by": current_user.id, "name": name[count], "location": "", "size": 0, "type": FileType.FOLDER.value}
+            )
             return cls.create_folder(file, file.id, name, count + 1)
 
     @classmethod
     @DB.connection_context()
     def is_parent_folder_exist(cls, parent_id):
+        # Check if parent folder exists
+        # Args:
+        #     parent_id: Parent folder ID
+        # Returns:
+        #     Boolean indicating if folder exists
         parent_files = cls.model.select().where(cls.model.id == parent_id)
         if parent_files.count():
             return True
@@ -157,9 +197,12 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_root_folder(cls, tenant_id):
-        for file in cls.model.select().where((cls.model.tenant_id == tenant_id),
-                                        (cls.model.parent_id == cls.model.id)
-                                        ):
+        # Get or create root folder for tenant
+        # Args:
+        #     tenant_id: Tenant ID
+        # Returns:
+        #     Root folder dictionary
+        for file in cls.model.select().where((cls.model.tenant_id == tenant_id), (cls.model.parent_id == cls.model.id)):
             return file.to_dict()
 
         file_id = get_uuid()
@@ -179,17 +222,29 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_kb_folder(cls, tenant_id):
-        for root in cls.model.select().where(
-                (cls.model.tenant_id == tenant_id), (cls.model.parent_id == cls.model.id)):
-            for folder in cls.model.select().where(
-                    (cls.model.tenant_id == tenant_id), (cls.model.parent_id == root.id),
-                    (cls.model.name == KNOWLEDGEBASE_FOLDER_NAME)):
+        # Get knowledge base folder for tenant
+        # Args:
+        #     tenant_id: Tenant ID
+        # Returns:
+        #     Knowledge base folder dictionary
+        for root in cls.model.select().where((cls.model.tenant_id == tenant_id), (cls.model.parent_id == cls.model.id)):
+            for folder in cls.model.select().where((cls.model.tenant_id == tenant_id), (cls.model.parent_id == root.id), (cls.model.name == KNOWLEDGEBASE_FOLDER_NAME)):
                 return folder.to_dict()
         assert False, "Can't find the KB folder. Database init error."
 
     @classmethod
     @DB.connection_context()
     def new_a_file_from_kb(cls, tenant_id, name, parent_id, ty=FileType.FOLDER.value, size=0, location=""):
+        # Create a new file from knowledge base
+        # Args:
+        #     tenant_id: Tenant ID
+        #     name: File name
+        #     parent_id: Parent folder ID
+        #     ty: File type
+        #     size: File size
+        #     location: File location
+        # Returns:
+        #     Created file dictionary
         for file in cls.query(tenant_id=tenant_id, parent_id=parent_id, name=name):
             return file.to_dict()
         file = {
@@ -201,7 +256,7 @@ class FileService(CommonService):
             "type": ty,
             "size": size,
             "location": location,
-            "source_type": FileSource.KNOWLEDGEBASE
+            "source_type": FileSource.KNOWLEDGEBASE,
         }
         cls.save(**file)
         return file
@@ -209,12 +264,15 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def init_knowledgebase_docs(cls, root_id, tenant_id):
-        for _ in cls.model.select().where((cls.model.name == KNOWLEDGEBASE_FOLDER_NAME)\
-                                          & (cls.model.parent_id == root_id)):
+        # Initialize knowledge base documents
+        # Args:
+        #     root_id: Root folder ID
+        #     tenant_id: Tenant ID
+        for _ in cls.model.select().where((cls.model.name == KNOWLEDGEBASE_FOLDER_NAME) & (cls.model.parent_id == root_id)):
             return
         folder = cls.new_a_file_from_kb(tenant_id, KNOWLEDGEBASE_FOLDER_NAME, root_id)
 
-        for kb in Knowledgebase.select(*[Knowledgebase.id, Knowledgebase.name]).where(Knowledgebase.tenant_id==tenant_id):
+        for kb in Knowledgebase.select(*[Knowledgebase.id, Knowledgebase.name]).where(Knowledgebase.tenant_id == tenant_id):
             kb_folder = cls.new_a_file_from_kb(tenant_id, kb.name, folder["id"])
             for doc in DocumentService.query(kb_id=kb.id):
                 FileService.add_file_from_kb(doc.to_dict(), kb_folder["id"], tenant_id)
@@ -222,6 +280,11 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_parent_folder(cls, file_id):
+        # Get parent folder of a file
+        # Args:
+        #     file_id: File ID
+        # Returns:
+        #     Parent folder object
         file = cls.model.select().where(cls.model.id == file_id)
         if file.count():
             e, file = cls.get_by_id(file[0].parent_id)
@@ -234,6 +297,11 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_all_parent_folders(cls, start_id):
+        # Get all parent folders in path
+        # Args:
+        #     start_id: Starting file ID
+        # Returns:
+        #     List of parent folder objects
         parent_folders = []
         current_id = start_id
         while current_id:
@@ -249,6 +317,11 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def insert(cls, file):
+        # Insert a new file record
+        # Args:
+        #     file: File data dictionary
+        # Returns:
+        #     Created file object
         if not cls.save(**file):
             raise RuntimeError("Database error (File)!")
         return File(**file)
@@ -256,6 +329,7 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def delete(cls, file):
+        #
         return cls.delete_by_id(file.id)
 
     @classmethod
@@ -267,12 +341,10 @@ class FileService(CommonService):
     @DB.connection_context()
     def delete_folder_by_pf_id(cls, user_id, folder_id):
         try:
-            files = cls.model.select().where((cls.model.tenant_id == user_id)
-                                             & (cls.model.parent_id == folder_id))
+            files = cls.model.select().where((cls.model.tenant_id == user_id) & (cls.model.parent_id == folder_id))
             for file in files:
                 cls.delete_folder_by_pf_id(user_id, file.id)
-            return cls.model.delete().where((cls.model.tenant_id == user_id)
-                                            & (cls.model.id == folder_id)).execute(),
+            return (cls.model.delete().where((cls.model.tenant_id == user_id) & (cls.model.id == folder_id)).execute(),)
         except Exception:
             logging.exception("delete_folder_by_pf_id")
             raise RuntimeError("Database error (File retrieval)!")
@@ -290,8 +362,7 @@ class FileService(CommonService):
 
         def dfs(parent_id):
             nonlocal size
-            for f in cls.model.select(*[cls.model.id, cls.model.size, cls.model.type]).where(
-                    cls.model.parent_id == parent_id, cls.model.id != parent_id):
+            for f in cls.model.select(*[cls.model.id, cls.model.size, cls.model.type]).where(cls.model.parent_id == parent_id, cls.model.id != parent_id):
                 size += f.size
                 if f.type == FileType.FOLDER.value:
                     dfs(f.id)
@@ -313,16 +384,16 @@ class FileService(CommonService):
             "type": doc["type"],
             "size": doc["size"],
             "location": doc["location"],
-            "source_type": FileSource.KNOWLEDGEBASE
+            "source_type": FileSource.KNOWLEDGEBASE,
         }
         cls.save(**file)
         File2DocumentService.save(**{"id": get_uuid(), "file_id": file["id"], "document_id": doc["id"]})
-    
+
     @classmethod
     @DB.connection_context()
     def move_file(cls, file_ids, folder_id):
         try:
-            cls.filter_update((cls.model.id << file_ids, ), { 'parent_id': folder_id })
+            cls.filter_update((cls.model.id << file_ids,), {"parent_id": folder_id})
         except Exception:
             logging.exception("move_file")
             raise RuntimeError("Database error (File move)!")
@@ -339,16 +410,13 @@ class FileService(CommonService):
         err, files = [], []
         for file in file_objs:
             try:
-                MAX_FILE_NUM_PER_USER = int(os.environ.get('MAX_FILE_NUM_PER_USER', 0))
+                MAX_FILE_NUM_PER_USER = int(os.environ.get("MAX_FILE_NUM_PER_USER", 0))
                 if MAX_FILE_NUM_PER_USER > 0 and DocumentService.get_doc_count(kb.tenant_id) >= MAX_FILE_NUM_PER_USER:
                     raise RuntimeError("Exceed the maximum file number of a free user!")
-                if len(file.filename) >= 128:
-                    raise RuntimeError("Exceed the maximum length of file name!")
+                if len(file.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+                    raise RuntimeError(f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.")
 
-                filename = duplicate_name(
-                    DocumentService.query,
-                    name=file.filename,
-                    kb_id=kb.id)
+                filename = duplicate_name(DocumentService.query, name=file.filename, kb_id=kb.id)
                 filetype = filename_type(filename)
                 if filetype == FileType.OTHER.value:
                     raise RuntimeError("This type of file has not been supported yet!")
@@ -356,15 +424,18 @@ class FileService(CommonService):
                 location = filename
                 while STORAGE_IMPL.obj_exist(kb.id, location):
                     location += "_"
+
                 blob = file.read()
+                if filetype == FileType.PDF.value:
+                    blob = read_potential_broken_pdf(blob)
                 STORAGE_IMPL.put(kb.id, location, blob)
 
                 doc_id = get_uuid()
 
                 img = thumbnail_img(filename, blob)
-                thumbnail_location = ''
+                thumbnail_location = ""
                 if img is not None:
-                    thumbnail_location = f'thumbnail_{doc_id}.png'
+                    thumbnail_location = f"thumbnail_{doc_id}.png"
                     STORAGE_IMPL.put(kb.id, thumbnail_location, img)
 
                 doc = {
@@ -377,7 +448,7 @@ class FileService(CommonService):
                     "name": filename,
                     "location": location,
                     "size": len(blob),
-                    "thumbnail": thumbnail_location
+                    "thumbnail": thumbnail_location,
                 }
                 DocumentService.insert(doc)
 
@@ -390,29 +461,17 @@ class FileService(CommonService):
 
     @staticmethod
     def parse_docs(file_objs, user_id):
-        from rag.app import presentation, picture, naive, audio, email
+        from rag.app import audio, email, naive, picture, presentation
 
         def dummy(prog=None, msg=""):
             pass
 
-        FACTORY = {
-            ParserType.PRESENTATION.value: presentation,
-            ParserType.PICTURE.value: picture,
-            ParserType.AUDIO.value: audio,
-            ParserType.EMAIL.value: email
-        }
+        FACTORY = {ParserType.PRESENTATION.value: presentation, ParserType.PICTURE.value: picture, ParserType.AUDIO.value: audio, ParserType.EMAIL.value: email}
         parser_config = {"chunk_token_num": 16096, "delimiter": "\n!?;。；！？", "layout_recognize": "Plain Text"}
         exe = ThreadPoolExecutor(max_workers=12)
         threads = []
         for file in file_objs:
-            kwargs = {
-                "lang": "English",
-                "callback": dummy,
-                "parser_config": parser_config,
-                "from_page": 0,
-                "to_page": 100000,
-                "tenant_id": user_id
-            }
+            kwargs = {"lang": "English", "callback": dummy, "parser_config": parser_config, "from_page": 0, "to_page": 100000, "tenant_id": user_id}
             filetype = filename_type(file.filename)
             blob = file.read()
             threads.append(exe.submit(FACTORY.get(FileService.get_parser(filetype, file.filename, ""), naive).chunk, file.filename, blob, **kwargs))
@@ -433,4 +492,4 @@ class FileService(CommonService):
             return ParserType.PRESENTATION.value
         if re.search(r"\.(eml)$", filename):
             return ParserType.EMAIL.value
-        return default
+        return default

api/db/services/knowledgebase_service.py

@@ -13,22 +13,115 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-from api.db import StatusEnum, TenantPermission
-from api.db.db_models import Knowledgebase, DB, Tenant, User, UserTenant,Document
-from api.db.services.common_service import CommonService
+from datetime import datetime
+
 from peewee import fn
 
+from api.db import StatusEnum, TenantPermission
+from api.db.db_models import DB, Document, Knowledgebase, Tenant, User, UserTenant
+from api.db.services.common_service import CommonService
+from api.utils import current_timestamp, datetime_format
+
 
 class KnowledgebaseService(CommonService):
+    """Service class for managing knowledge base operations.
+
+    This class extends CommonService to provide specialized functionality for knowledge base
+    management, including document parsing status tracking, access control, and configuration
+    management. It handles operations such as listing, creating, updating, and deleting
+    knowledge bases, as well as managing their associated documents and permissions.
+
+    The class implements a comprehensive set of methods for:
+    - Document parsing status verification
+    - Knowledge base access control
+    - Parser configuration management
+    - Tenant-based knowledge base organization
+
+    Attributes:
+        model: The Knowledgebase model class for database operations.
+    """
     model = Knowledgebase
 
     @classmethod
     @DB.connection_context()
-    def list_documents_by_ids(cls,kb_ids):
-        doc_ids=cls.model.select(Document.id.alias("document_id")).join(Document,on=(cls.model.id == Document.kb_id)).where(
+    def accessible4deletion(cls, kb_id, user_id):
+        """Check if a knowledge base can be deleted by a specific user.
+
+        This method verifies whether a user has permission to delete a knowledge base
+        by checking if they are the creator of that knowledge base.
+
+        Args:
+            kb_id (str): The unique identifier of the knowledge base to check.
+            user_id (str): The unique identifier of the user attempting the deletion.
+
+        Returns:
+            bool: True if the user has permission to delete the knowledge base,
+                  False if the user doesn't have permission or the knowledge base doesn't exist.
+
+        Example:
+            >>> KnowledgebaseService.accessible4deletion("kb123", "user456")
+            True
+
+        Note:
+            - This method only checks creator permissions
+            - A return value of False can mean either:
+                1. The knowledge base doesn't exist
+                2. The user is not the creator of the knowledge base
+        """
+        # Check if a knowledge base can be deleted by a user
+        docs = cls.model.select(
+            cls.model.id).where(cls.model.id == kb_id, cls.model.created_by == user_id).paginate(0, 1)
+        docs = docs.dicts()
+        if not docs:
+            return False
+        return True
+
+    @classmethod
+    @DB.connection_context()
+    def is_parsed_done(cls, kb_id):
+        # Check if all documents in the knowledge base have completed parsing
+        #
+        # Args:
+        #     kb_id: Knowledge base ID
+        #
+        # Returns:
+        #     If all documents are parsed successfully, returns (True, None)
+        #     If any document is not fully parsed, returns (False, error_message)
+        from api.db import TaskStatus
+        from api.db.services.document_service import DocumentService
+
+        # Get knowledge base information
+        kbs = cls.query(id=kb_id)
+        if not kbs:
+            return False, "Knowledge base not found"
+        kb = kbs[0]
+
+        # Get all documents in the knowledge base
+        docs, _ = DocumentService.get_by_kb_id(kb_id, 1, 1000, "create_time", True, "", [], [])
+
+        # Check parsing status of each document
+        for doc in docs:
+            # If document is being parsed, don't allow chat creation
+            if doc['run'] == TaskStatus.RUNNING.value or doc['run'] == TaskStatus.CANCEL.value or doc['run'] == TaskStatus.FAIL.value:
+                return False, f"Document '{doc['name']}' in dataset '{kb.name}' is still being parsed. Please wait until all documents are parsed before starting a chat."
+            # If document is not yet parsed and has no chunks, don't allow chat creation
+            if doc['run'] == TaskStatus.UNSTART.value and doc['chunk_num'] == 0:
+                return False, f"Document '{doc['name']}' in dataset '{kb.name}' has not been parsed yet. Please parse all documents before starting a chat."
+
+        return True, None
+
+    @classmethod
+    @DB.connection_context()
+    def list_documents_by_ids(cls, kb_ids):
+        # Get document IDs associated with given knowledge base IDs
+        # Args:
+        #     kb_ids: List of knowledge base IDs
+        # Returns:
+        #     List of document IDs
+        doc_ids = cls.model.select(Document.id.alias("document_id")).join(Document, on=(cls.model.id == Document.kb_id)).where(
             cls.model.id.in_(kb_ids)
         )
-        doc_ids =list(doc_ids.dicts())
+        doc_ids = list(doc_ids.dicts())
         doc_ids = [doc["document_id"] for doc in doc_ids]
         return doc_ids
 
@@ -39,12 +132,25 @@ class KnowledgebaseService(CommonService):
                           orderby, desc, keywords,
                           parser_id=None
                           ):
+        # Get knowledge bases by tenant IDs with pagination and filtering
+        # Args:
+        #     joined_tenant_ids: List of tenant IDs
+        #     user_id: Current user ID
+        #     page_number: Page number for pagination
+        #     items_per_page: Number of items per page
+        #     orderby: Field to order by
+        #     desc: Boolean indicating descending order
+        #     keywords: Search keywords
+        #     parser_id: Optional parser ID filter
+        # Returns:
+        #     Tuple of (knowledge_base_list, total_count)
         fields = [
             cls.model.id,
             cls.model.avatar,
             cls.model.name,
             cls.model.language,
             cls.model.description,
+            cls.model.tenant_id,
             cls.model.permission,
             cls.model.doc_num,
             cls.model.token_num,
@@ -79,13 +185,19 @@ class KnowledgebaseService(CommonService):
 
         count = kbs.count()
 
-        kbs = kbs.paginate(page_number, items_per_page)
+        if page_number and items_per_page:
+            kbs = kbs.paginate(page_number, items_per_page)
 
         return list(kbs.dicts()), count
 
     @classmethod
     @DB.connection_context()
     def get_kb_ids(cls, tenant_id):
+        # Get all knowledge base IDs for a tenant
+        # Args:
+        #     tenant_id: Tenant ID
+        # Returns:
+        #     List of knowledge base IDs
         fields = [
             cls.model.id,
         ]
@@ -96,9 +208,13 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_detail(cls, kb_id):
+        # Get detailed information about a knowledge base
+        # Args:
+        #     kb_id: Knowledge base ID
+        # Returns:
+        #     Dictionary containing knowledge base details
         fields = [
             cls.model.id,
-            # Tenant.embd_id,
             cls.model.embd_id,
             cls.model.avatar,
             cls.model.name,
@@ -110,26 +226,33 @@ class KnowledgebaseService(CommonService):
             cls.model.chunk_num,
             cls.model.parser_id,
             cls.model.parser_config,
-            cls.model.pagerank]
+            cls.model.pagerank,
+            cls.model.create_time,
+            cls.model.update_time
+            ]
         kbs = cls.model.select(*fields).join(Tenant, on=(
-                (Tenant.id == cls.model.tenant_id) & (Tenant.status == StatusEnum.VALID.value))).where(
+            (Tenant.id == cls.model.tenant_id) & (Tenant.status == StatusEnum.VALID.value))).where(
             (cls.model.id == kb_id),
             (cls.model.status == StatusEnum.VALID.value)
         )
         if not kbs:
             return
         d = kbs[0].to_dict()
-        # d["embd_id"] = kbs[0].tenant.embd_id
         return d
 
     @classmethod
     @DB.connection_context()
     def update_parser_config(cls, id, config):
+        # Update parser configuration for a knowledge base
+        # Args:
+        #     id: Knowledge base ID
+        #     config: New parser configuration
         e, m = cls.get_by_id(id)
         if not e:
             raise LookupError(f"knowledgebase({id}) not found.")
 
         def dfs_update(old, new):
+            # Deep update of nested configuration
             for k, v in new.items():
                 if k not in old:
                     old[k] = v
@@ -146,9 +269,24 @@ class KnowledgebaseService(CommonService):
         dfs_update(m.parser_config, config)
         cls.update_by_id(id, {"parser_config": m.parser_config})
 
+    @classmethod
+    @DB.connection_context()
+    def delete_field_map(cls, id):
+        e, m = cls.get_by_id(id)
+        if not e:
+            raise LookupError(f"knowledgebase({id}) not found.")
+
+        m.parser_config.pop("field_map", None)
+        cls.update_by_id(id, {"parser_config": m.parser_config})
+
     @classmethod
     @DB.connection_context()
     def get_field_map(cls, ids):
+        # Get field mappings for knowledge bases
+        # Args:
+        #     ids: List of knowledge base IDs
+        # Returns:
+        #     Dictionary of field mappings
         conf = {}
         for k in cls.get_by_ids(ids):
             if k.parser_config and "field_map" in k.parser_config:
@@ -158,6 +296,12 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_by_name(cls, kb_name, tenant_id):
+        # Get knowledge base by name and tenant ID
+        # Args:
+        #     kb_name: Knowledge base name
+        #     tenant_id: Tenant ID
+        # Returns:
+        #     Tuple of (exists, knowledge_base)
         kb = cls.model.select().where(
             (cls.model.name == kb_name)
             & (cls.model.tenant_id == tenant_id)
@@ -170,12 +314,27 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_all_ids(cls):
+        # Get all knowledge base IDs
+        # Returns:
+        #     List of all knowledge base IDs
         return [m["id"] for m in cls.model.select(cls.model.id).dicts()]
 
     @classmethod
     @DB.connection_context()
     def get_list(cls, joined_tenant_ids, user_id,
                  page_number, items_per_page, orderby, desc, id, name):
+        # Get list of knowledge bases with filtering and pagination
+        # Args:
+        #     joined_tenant_ids: List of tenant IDs
+        #     user_id: Current user ID
+        #     page_number: Page number for pagination
+        #     items_per_page: Number of items per page
+        #     orderby: Field to order by
+        #     desc: Boolean indicating descending order
+        #     id: Optional ID filter
+        #     name: Optional name filter
+        # Returns:
+        #     List of knowledge bases
         kbs = cls.model.select()
         if id:
             kbs = kbs.where(cls.model.id == id)
@@ -184,7 +343,7 @@ class KnowledgebaseService(CommonService):
         kbs = kbs.where(
             ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
                                                             TenantPermission.TEAM.value)) | (
-                     cls.model.tenant_id == user_id))
+                cls.model.tenant_id == user_id))
             & (cls.model.status == StatusEnum.VALID.value)
         )
         if desc:
@@ -199,9 +358,15 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def accessible(cls, kb_id, user_id):
+        # Check if a knowledge base is accessible by a user
+        # Args:
+        #     kb_id: Knowledge base ID
+        #     user_id: User ID
+        # Returns:
+        #     Boolean indicating accessibility
         docs = cls.model.select(
             cls.model.id).join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
-            ).where(cls.model.id == kb_id, UserTenant.user_id == user_id).paginate(0, 1)
+                               ).where(cls.model.id == kb_id, UserTenant.user_id == user_id).paginate(0, 1)
         docs = docs.dicts()
         if not docs:
             return False
@@ -210,26 +375,64 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_kb_by_id(cls, kb_id, user_id):
+        # Get knowledge base by ID and user ID
+        # Args:
+        #     kb_id: Knowledge base ID
+        #     user_id: User ID
+        # Returns:
+        #     List containing knowledge base information
         kbs = cls.model.select().join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
-            ).where(cls.model.id == kb_id, UserTenant.user_id == user_id).paginate(0, 1)
+                                      ).where(cls.model.id == kb_id, UserTenant.user_id == user_id).paginate(0, 1)
         kbs = kbs.dicts()
         return list(kbs)
 
     @classmethod
     @DB.connection_context()
     def get_kb_by_name(cls, kb_name, user_id):
+        # Get knowledge base by name and user ID
+        # Args:
+        #     kb_name: Knowledge base name
+        #     user_id: User ID
+        # Returns:
+        #     List containing knowledge base information
         kbs = cls.model.select().join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
-            ).where(cls.model.name == kb_name, UserTenant.user_id == user_id).paginate(0, 1)
+                                      ).where(cls.model.name == kb_name, UserTenant.user_id == user_id).paginate(0, 1)
         kbs = kbs.dicts()
         return list(kbs)
 
     @classmethod
     @DB.connection_context()
-    def accessible4deletion(cls, kb_id, user_id):
-        docs = cls.model.select(
-            cls.model.id).where(cls.model.id == kb_id, cls.model.created_by == user_id).paginate(0, 1)
-        docs = docs.dicts()
-        if not docs:
-            return False
-        return True
+    def atomic_increase_doc_num_by_id(cls, kb_id):
+        data = {}
+        data["update_time"] = current_timestamp()
+        data["update_date"] = datetime_format(datetime.now())
+        data["doc_num"] = cls.model.doc_num + 1
+        num = cls.model.update(data).where(cls.model.id == kb_id).execute()
+        return num
+
+    @classmethod
+    @DB.connection_context()
+    def update_document_number_in_init(cls, kb_id, doc_num):
+        """
+        Only use this function when init system
+        """
+        ok, kb = cls.get_by_id(kb_id)
+        if not ok:
+            return
+        kb.doc_num = doc_num
+
+        dirty_fields = kb.dirty_fields
+        if cls.model._meta.combined.get("update_time") in dirty_fields:
+            dirty_fields.remove(cls.model._meta.combined["update_time"])
+
+        if cls.model._meta.combined.get("update_date") in dirty_fields:
+            dirty_fields.remove(cls.model._meta.combined["update_date"])
+
+        try:
+            kb.save(only=dirty_fields)
+        except ValueError as e:
+            if str(e) == "no data to save!":
+                pass # that's OK
+            else:
+                raise e
 

api/db/services/langfuse_service.py

@@ -0,0 +1,71 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from datetime import datetime
+
+import peewee
+
+from api.db.db_models import DB, TenantLangfuse
+from api.db.services.common_service import CommonService
+from api.utils import current_timestamp, datetime_format
+
+
+class TenantLangfuseService(CommonService):
+    """
+    All methods that modify the status should be enclosed within a DB.atomic() context to ensure atomicity
+    and maintain data integrity in case of errors during execution.
+    """
+
+    model = TenantLangfuse
+
+    @classmethod
+    @DB.connection_context()
+    def filter_by_tenant(cls, tenant_id):
+        fields = [cls.model.tenant_id, cls.model.host, cls.model.secret_key, cls.model.public_key]
+        try:
+            keys = cls.model.select(*fields).where(cls.model.tenant_id == tenant_id).first()
+            return keys
+        except peewee.DoesNotExist:
+            return None
+
+    @classmethod
+    @DB.connection_context()
+    def filter_by_tenant_with_info(cls, tenant_id):
+        fields = [cls.model.tenant_id, cls.model.host, cls.model.secret_key, cls.model.public_key]
+        try:
+            keys = cls.model.select(*fields).where(cls.model.tenant_id == tenant_id).dicts().first()
+            return keys
+        except peewee.DoesNotExist:
+            return None
+
+    @classmethod
+    def update_by_tenant(cls, tenant_id, langfuse_keys):
+        langfuse_keys["update_time"] = current_timestamp()
+        langfuse_keys["update_date"] = datetime_format(datetime.now())
+        return cls.model.update(**langfuse_keys).where(cls.model.tenant_id == tenant_id).execute()
+
+    @classmethod
+    def save(cls, **kwargs):
+        kwargs["create_time"] = current_timestamp()
+        kwargs["create_date"] = datetime_format(datetime.now())
+        kwargs["update_time"] = current_timestamp()
+        kwargs["update_date"] = datetime_format(datetime.now())
+        obj = cls.model.create(**kwargs)
+        return obj
+
+    @classmethod
+    def delete_model(cls, langfuse_model):
+        langfuse_model.delete_instance()

api/db/services/llm_service.py

@@ -13,17 +13,17 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import json
 import logging
-import os
 
-from api.db.services.user_service import TenantService
-from api.utils.file_utils import get_project_base_directory
-from rag.llm import EmbeddingModel, CvModel, ChatModel, RerankModel, Seq2txtModel, TTSModel
+from langfuse import Langfuse
+
+from api import settings
 from api.db import LLMType
-from api.db.db_models import DB
-from api.db.db_models import LLMFactories, LLM, TenantLLM
+from api.db.db_models import DB, LLM, LLMFactories, TenantLLM
 from api.db.services.common_service import CommonService
+from api.db.services.langfuse_service import TenantLangfuseService
+from api.db.services.user_service import TenantService
+from rag.llm import ChatModel, CvModel, EmbeddingModel, RerankModel, Seq2txtModel, TTSModel
 
 
 class LLMFactoriesService(CommonService):
@@ -52,16 +52,8 @@ class TenantLLMService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_my_llms(cls, tenant_id):
-        fields = [
-            cls.model.llm_factory,
-            LLMFactories.logo,
-            LLMFactories.tags,
-            cls.model.model_type,
-            cls.model.llm_name,
-            cls.model.used_tokens
-        ]
-        objs = cls.model.select(*fields).join(LLMFactories, on=(cls.model.llm_factory == LLMFactories.name)).where(
-            cls.model.tenant_id == tenant_id, ~cls.model.api_key.is_null()).dicts()
+        fields = [cls.model.llm_factory, LLMFactories.logo, LLMFactories.tags, cls.model.model_type, cls.model.llm_name, cls.model.used_tokens]
+        objs = cls.model.select(*fields).join(LLMFactories, on=(cls.model.llm_factory == LLMFactories.name)).where(cls.model.tenant_id == tenant_id, ~cls.model.api_key.is_null()).dicts()
 
         return list(objs)
 
@@ -75,7 +67,7 @@ class TenantLLMService(CommonService):
 
         # model name must be xxx@yyy
         try:
-            model_factories = json.load(open(os.path.join(get_project_base_directory(), "conf/llm_factories.json"), "r"))["factory_llm_infos"]
+            model_factories = settings.FACTORY_LLM_INFOS
             model_providers = set([f["name"] for f in model_factories])
             if arr[-1] not in model_providers:
                 return model_name, None
@@ -86,8 +78,7 @@ class TenantLLMService(CommonService):
 
     @classmethod
     @DB.connection_context()
-    def model_instance(cls, tenant_id, llm_type,
-                       llm_name=None, lang="Chinese"):
+    def get_model_config(cls, tenant_id, llm_type, llm_name=None):
         e, tenant = TenantService.get_by_id(tenant_id)
         if not e:
             raise LookupError("Tenant not found")
@@ -109,8 +100,15 @@ class TenantLLMService(CommonService):
 
         model_config = cls.get_api_key(tenant_id, mdlnm)
         mdlnm, fid = TenantLLMService.split_model_name_and_factory(mdlnm)
+        if not model_config:  # for some cases seems fid mismatch
+            model_config = cls.get_api_key(tenant_id, mdlnm)
         if model_config:
             model_config = model_config.to_dict()
+            llm = LLMService.query(llm_name=mdlnm) if not fid else LLMService.query(llm_name=mdlnm, fid=fid)
+            if not llm and fid:  # for some cases seems fid mismatch
+                llm = LLMService.query(llm_name=mdlnm)
+            if llm:
+                model_config["is_tools"] = llm[0].is_tools
         if not model_config:
             if llm_type in [LLMType.EMBEDDING, LLMType.RERANK]:
                 llm = LLMService.query(llm_name=mdlnm) if not fid else LLMService.query(llm_name=mdlnm, fid=fid)
@@ -118,47 +116,41 @@ class TenantLLMService(CommonService):
                     model_config = {"llm_factory": llm[0].fid, "api_key": "", "llm_name": mdlnm, "api_base": ""}
             if not model_config:
                 if mdlnm == "flag-embedding":
-                    model_config = {"llm_factory": "Tongyi-Qianwen", "api_key": "",
-                                    "llm_name": llm_name, "api_base": ""}
+                    model_config = {"llm_factory": "Tongyi-Qianwen", "api_key": "", "llm_name": llm_name, "api_base": ""}
                 else:
                     if not mdlnm:
                         raise LookupError(f"Type of {llm_type} model is not set.")
                     raise LookupError("Model({}) not authorized".format(mdlnm))
+        return model_config
 
+    @classmethod
+    @DB.connection_context()
+    def model_instance(cls, tenant_id, llm_type, llm_name=None, lang="Chinese"):
+        model_config = TenantLLMService.get_model_config(tenant_id, llm_type, llm_name)
         if llm_type == LLMType.EMBEDDING.value:
             if model_config["llm_factory"] not in EmbeddingModel:
                 return
-            return EmbeddingModel[model_config["llm_factory"]](
-                model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
+            return EmbeddingModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
 
         if llm_type == LLMType.RERANK:
             if model_config["llm_factory"] not in RerankModel:
                 return
-            return RerankModel[model_config["llm_factory"]](
-                model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
+            return RerankModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
 
         if llm_type == LLMType.IMAGE2TEXT.value:
             if model_config["llm_factory"] not in CvModel:
                 return
-            return CvModel[model_config["llm_factory"]](
-                model_config["api_key"], model_config["llm_name"], lang,
-                base_url=model_config["api_base"]
-            )
+            return CvModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], lang, base_url=model_config["api_base"])
 
         if llm_type == LLMType.CHAT.value:
             if model_config["llm_factory"] not in ChatModel:
                 return
-            return ChatModel[model_config["llm_factory"]](
-                model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
+            return ChatModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], base_url=model_config["api_base"])
 
         if llm_type == LLMType.SPEECH2TEXT:
             if model_config["llm_factory"] not in Seq2txtModel:
                 return
-            return Seq2txtModel[model_config["llm_factory"]](
-                key=model_config["api_key"], model_name=model_config["llm_name"],
-                lang=lang,
-                base_url=model_config["api_base"]
-            )
+            return Seq2txtModel[model_config["llm_factory"]](key=model_config["api_key"], model_name=model_config["llm_name"], lang=lang, base_url=model_config["api_base"])
         if llm_type == LLMType.TTS:
             if model_config["llm_factory"] not in TTSModel:
                 return
@@ -173,133 +165,220 @@ class TenantLLMService(CommonService):
     def increase_usage(cls, tenant_id, llm_type, used_tokens, llm_name=None):
         e, tenant = TenantService.get_by_id(tenant_id)
         if not e:
-            raise LookupError("Tenant not found")
+            logging.error(f"Tenant not found: {tenant_id}")
+            return 0
 
-        if llm_type == LLMType.EMBEDDING.value:
-            mdlnm = tenant.embd_id
-        elif llm_type == LLMType.SPEECH2TEXT.value:
-            mdlnm = tenant.asr_id
-        elif llm_type == LLMType.IMAGE2TEXT.value:
-            mdlnm = tenant.img2txt_id
-        elif llm_type == LLMType.CHAT.value:
-            mdlnm = tenant.llm_id if not llm_name else llm_name
-        elif llm_type == LLMType.RERANK:
-            mdlnm = tenant.rerank_id if not llm_name else llm_name
-        elif llm_type == LLMType.TTS:
-            mdlnm = tenant.tts_id if not llm_name else llm_name
-        else:
-            assert False, "LLM type error"
+        llm_map = {
+            LLMType.EMBEDDING.value: tenant.embd_id if not llm_name else llm_name,
+            LLMType.SPEECH2TEXT.value: tenant.asr_id,
+            LLMType.IMAGE2TEXT.value: tenant.img2txt_id,
+            LLMType.CHAT.value: tenant.llm_id if not llm_name else llm_name,
+            LLMType.RERANK.value: tenant.rerank_id if not llm_name else llm_name,
+            LLMType.TTS.value: tenant.tts_id if not llm_name else llm_name,
+        }
+
+        mdlnm = llm_map.get(llm_type)
+        if mdlnm is None:
+            logging.error(f"LLM type error: {llm_type}")
+            return 0
 
         llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(mdlnm)
 
-        num = 0
         try:
-            if llm_factory:
-                tenant_llms = cls.query(tenant_id=tenant_id, llm_name=llm_name, llm_factory=llm_factory)
-            else:
-                tenant_llms = cls.query(tenant_id=tenant_id, llm_name=llm_name)
-            if not tenant_llms:
-                return num
-            else:
-                tenant_llm = tenant_llms[0]
-                num = cls.model.update(used_tokens=tenant_llm.used_tokens + used_tokens) \
-                    .where(cls.model.tenant_id == tenant_id, cls.model.llm_factory == tenant_llm.llm_factory, cls.model.llm_name == llm_name) \
-                    .execute()
+            num = (
+                cls.model.update(used_tokens=cls.model.used_tokens + used_tokens)
+                .where(cls.model.tenant_id == tenant_id, cls.model.llm_name == llm_name, cls.model.llm_factory == llm_factory if llm_factory else True)
+                .execute()
+            )
         except Exception:
-            logging.exception("TenantLLMService.increase_usage got exception")
+            logging.exception("TenantLLMService.increase_usage got exception,Failed to update used_tokens for tenant_id=%s, llm_name=%s", tenant_id, llm_name)
+            return 0
+
         return num
 
     @classmethod
     @DB.connection_context()
     def get_openai_models(cls):
-        objs = cls.model.select().where(
-            (cls.model.llm_factory == "OpenAI"),
-            ~(cls.model.llm_name == "text-embedding-3-small"),
-            ~(cls.model.llm_name == "text-embedding-3-large")
-        ).dicts()
+        objs = cls.model.select().where((cls.model.llm_factory == "OpenAI"), ~(cls.model.llm_name == "text-embedding-3-small"), ~(cls.model.llm_name == "text-embedding-3-large")).dicts()
         return list(objs)
 
 
-class LLMBundle(object):
+class LLMBundle:
     def __init__(self, tenant_id, llm_type, llm_name=None, lang="Chinese"):
         self.tenant_id = tenant_id
         self.llm_type = llm_type
         self.llm_name = llm_name
-        self.mdl = TenantLLMService.model_instance(
-            tenant_id, llm_type, llm_name, lang=lang)
-        assert self.mdl, "Can't find model for {}/{}/{}".format(
-            tenant_id, llm_type, llm_name)
-        self.max_length = 8192
-        for lm in LLMService.query(llm_name=llm_name):
-            self.max_length = lm.max_tokens
-            break
+        self.mdl = TenantLLMService.model_instance(tenant_id, llm_type, llm_name, lang=lang)
+        assert self.mdl, "Can't find model for {}/{}/{}".format(tenant_id, llm_type, llm_name)
+        model_config = TenantLLMService.get_model_config(tenant_id, llm_type, llm_name)
+        self.max_length = model_config.get("max_tokens", 8192)
+
+        self.is_tools = model_config.get("is_tools", False)
+
+        langfuse_keys = TenantLangfuseService.filter_by_tenant(tenant_id=tenant_id)
+        if langfuse_keys:
+            langfuse = Langfuse(public_key=langfuse_keys.public_key, secret_key=langfuse_keys.secret_key, host=langfuse_keys.host)
+            if langfuse.auth_check():
+                self.langfuse = langfuse
+                self.trace = self.langfuse.trace(name=f"{self.llm_type}-{self.llm_name}")
+        else:
+            self.langfuse = None
+
+    def bind_tools(self, toolcall_session, tools):
+        if not self.is_tools:
+            logging.warning(f"Model {self.llm_name} does not support tool call, but you have assigned one or more tools to it!")
+            return
+        self.mdl.bind_tools(toolcall_session, tools)
 
     def encode(self, texts: list):
+        if self.langfuse:
+            generation = self.trace.generation(name="encode", model=self.llm_name, input={"texts": texts})
+
         embeddings, used_tokens = self.mdl.encode(texts)
-        if not TenantLLMService.increase_usage(
-                self.tenant_id, self.llm_type, used_tokens):
-            logging.error(
-                "LLMBundle.encode can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
+        llm_name = getattr(self, "llm_name", None)
+        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens, llm_name):
+            logging.error("LLMBundle.encode can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
+
+        if self.langfuse:
+            generation.end(usage_details={"total_tokens": used_tokens})
+
         return embeddings, used_tokens
 
     def encode_queries(self, query: str):
+        if self.langfuse:
+            generation = self.trace.generation(name="encode_queries", model=self.llm_name, input={"query": query})
+
         emd, used_tokens = self.mdl.encode_queries(query)
-        if not TenantLLMService.increase_usage(
-                self.tenant_id, self.llm_type, used_tokens):
-            logging.error(
-                "LLMBundle.encode_queries can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
+        llm_name = getattr(self, "llm_name", None)
+        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens, llm_name):
+            logging.error("LLMBundle.encode_queries can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
+
+        if self.langfuse:
+            generation.end(usage_details={"total_tokens": used_tokens})
+
         return emd, used_tokens
 
     def similarity(self, query: str, texts: list):
+        if self.langfuse:
+            generation = self.trace.generation(name="similarity", model=self.llm_name, input={"query": query, "texts": texts})
+
         sim, used_tokens = self.mdl.similarity(query, texts)
-        if not TenantLLMService.increase_usage(
-                self.tenant_id, self.llm_type, used_tokens):
-            logging.error(
-                "LLMBundle.similarity can't update token usage for {}/RERANK used_tokens: {}".format(self.tenant_id, used_tokens))
+        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens):
+            logging.error("LLMBundle.similarity can't update token usage for {}/RERANK used_tokens: {}".format(self.tenant_id, used_tokens))
+
+        if self.langfuse:
+            generation.end(usage_details={"total_tokens": used_tokens})
+
         return sim, used_tokens
 
     def describe(self, image, max_tokens=300):
-        txt, used_tokens = self.mdl.describe(image, max_tokens)
-        if not TenantLLMService.increase_usage(
-                self.tenant_id, self.llm_type, used_tokens):
-            logging.error(
-                "LLMBundle.describe can't update token usage for {}/IMAGE2TEXT used_tokens: {}".format(self.tenant_id, used_tokens))
+        if self.langfuse:
+            generation = self.trace.generation(name="describe", metadata={"model": self.llm_name})
+
+        txt, used_tokens = self.mdl.describe(image)
+        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens):
+            logging.error("LLMBundle.describe can't update token usage for {}/IMAGE2TEXT used_tokens: {}".format(self.tenant_id, used_tokens))
+
+        if self.langfuse:
+            generation.end(output={"output": txt}, usage_details={"total_tokens": used_tokens})
+
+        return txt
+
+    def describe_with_prompt(self, image, prompt):
+        if self.langfuse:
+            generation = self.trace.generation(name="describe_with_prompt", metadata={"model": self.llm_name, "prompt": prompt})
+
+        txt, used_tokens = self.mdl.describe_with_prompt(image, prompt)
+        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens):
+            logging.error("LLMBundle.describe can't update token usage for {}/IMAGE2TEXT used_tokens: {}".format(self.tenant_id, used_tokens))
+
+        if self.langfuse:
+            generation.end(output={"output": txt}, usage_details={"total_tokens": used_tokens})
+
         return txt
 
     def transcription(self, audio):
+        if self.langfuse:
+            generation = self.trace.generation(name="transcription", metadata={"model": self.llm_name})
+
         txt, used_tokens = self.mdl.transcription(audio)
-        if not TenantLLMService.increase_usage(
-                self.tenant_id, self.llm_type, used_tokens):
-            logging.error(
-                "LLMBundle.transcription can't update token usage for {}/SEQUENCE2TXT used_tokens: {}".format(self.tenant_id, used_tokens))
+        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens):
+            logging.error("LLMBundle.transcription can't update token usage for {}/SEQUENCE2TXT used_tokens: {}".format(self.tenant_id, used_tokens))
+
+        if self.langfuse:
+            generation.end(output={"output": txt}, usage_details={"total_tokens": used_tokens})
+
         return txt
 
     def tts(self, text):
+        if self.langfuse:
+            span = self.trace.span(name="tts", input={"text": text})
+
         for chunk in self.mdl.tts(text):
             if isinstance(chunk, int):
-                if not TenantLLMService.increase_usage(
-                        self.tenant_id, self.llm_type, chunk, self.llm_name):
-                    logging.error(
-                        "LLMBundle.tts can't update token usage for {}/TTS".format(self.tenant_id))
+                if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, chunk, self.llm_name):
+                    logging.error("LLMBundle.tts can't update token usage for {}/TTS".format(self.tenant_id))
                 return
             yield chunk
 
+        if self.langfuse:
+            span.end()
+
+    def _remove_reasoning_content(self, txt: str) -> str:
+        first_think_start = txt.find("<think>")
+        if first_think_start == -1:
+            return txt
+
+        last_think_end = txt.rfind("</think>")
+        if last_think_end == -1:
+            return txt
+
+        if last_think_end < first_think_start:
+            return txt
+
+        return txt[last_think_end + len("</think>") :]
+
     def chat(self, system, history, gen_conf):
-        txt, used_tokens = self.mdl.chat(system, history, gen_conf)
-        if isinstance(txt, int) and not TenantLLMService.increase_usage(
-                self.tenant_id, self.llm_type, used_tokens, self.llm_name):
-            logging.error(
-                "LLMBundle.chat can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name,
-                                                                                                           used_tokens))
+        if self.langfuse:
+            generation = self.trace.generation(name="chat", model=self.llm_name, input={"system": system, "history": history})
+
+        chat = self.mdl.chat
+        if self.is_tools and self.mdl.is_tools:
+            chat = self.mdl.chat_with_tools
+
+        txt, used_tokens = chat(system, history, gen_conf)
+        txt = self._remove_reasoning_content(txt)
+
+        if isinstance(txt, int) and not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens, self.llm_name):
+            logging.error("LLMBundle.chat can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name, used_tokens))
+
+        if self.langfuse:
+            generation.end(output={"output": txt}, usage_details={"total_tokens": used_tokens})
+
         return txt
 
     def chat_streamly(self, system, history, gen_conf):
-        for txt in self.mdl.chat_streamly(system, history, gen_conf):
+        if self.langfuse:
+            generation = self.trace.generation(name="chat_streamly", model=self.llm_name, input={"system": system, "history": history})
+
+        ans = ""
+        chat_streamly = self.mdl.chat_streamly
+        total_tokens = 0
+        if self.is_tools and self.mdl.is_tools:
+            chat_streamly = self.mdl.chat_streamly_with_tools
+
+        for txt in chat_streamly(system, history, gen_conf):
             if isinstance(txt, int):
-                if not TenantLLMService.increase_usage(
-                        self.tenant_id, self.llm_type, txt, self.llm_name):
-                    logging.error(
-                        "LLMBundle.chat_streamly can't update token usage for {}/CHAT llm_name: {}, content: {}".format(self.tenant_id, self.llm_name,
-                                                                                                                        txt))
-                return
-            yield txt
+                total_tokens = txt
+                if self.langfuse:
+                    generation.end(output={"output": ans})
+                break
+
+            if txt.endswith("</think>"):
+                ans = ans.rstrip("</think>")
+
+            ans += txt
+            yield ans
+        if total_tokens > 0:
+            if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, txt, self.llm_name):
+                logging.error("LLMBundle.chat_streamly can't update token usage for {}/CHAT llm_name: {}, content: {}".format(self.tenant_id, self.llm_name, txt))

api/db/services/search_service.py

@@ -0,0 +1,110 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from datetime import datetime
+
+from peewee import fn
+
+from api.db import StatusEnum
+from api.db.db_models import DB, Search, User
+from api.db.services.common_service import CommonService
+from api.utils import current_timestamp, datetime_format
+
+
+class SearchService(CommonService):
+    model = Search
+
+    @classmethod
+    def save(cls, **kwargs):
+        kwargs["create_time"] = current_timestamp()
+        kwargs["create_date"] = datetime_format(datetime.now())
+        kwargs["update_time"] = current_timestamp()
+        kwargs["update_date"] = datetime_format(datetime.now())
+        obj = cls.model.create(**kwargs)
+        return obj
+
+    @classmethod
+    @DB.connection_context()
+    def accessible4deletion(cls, search_id, user_id) -> bool:
+        search = (
+            cls.model.select(cls.model.id)
+            .where(
+                cls.model.id == search_id,
+                cls.model.created_by == user_id,
+                cls.model.status == StatusEnum.VALID.value,
+            )
+            .first()
+        )
+        return search is not None
+
+    @classmethod
+    @DB.connection_context()
+    def get_detail(cls, search_id):
+        fields = [
+            cls.model.id,
+            cls.model.avatar,
+            cls.model.tenant_id,
+            cls.model.name,
+            cls.model.description,
+            cls.model.created_by,
+            cls.model.search_config,
+            cls.model.update_time,
+            User.nickname,
+            User.avatar.alias("tenant_avatar"),
+        ]
+        search = (
+            cls.model.select(*fields)
+            .join(User, on=((User.id == cls.model.tenant_id) & (User.status == StatusEnum.VALID.value)))
+            .where((cls.model.id == search_id) & (cls.model.status == StatusEnum.VALID.value))
+            .first()
+            .to_dict()
+        )
+        return search
+
+    @classmethod
+    @DB.connection_context()
+    def get_by_tenant_ids(cls, joined_tenant_ids, user_id, page_number, items_per_page, orderby, desc, keywords):
+        fields = [
+            cls.model.id,
+            cls.model.avatar,
+            cls.model.tenant_id,
+            cls.model.name,
+            cls.model.description,
+            cls.model.created_by,
+            cls.model.status,
+            cls.model.update_time,
+            cls.model.create_time,
+            User.nickname,
+            User.avatar.alias("tenant_avatar"),
+        ]
+        query = (
+            cls.model.select(*fields)
+            .join(User, on=(cls.model.tenant_id == User.id))
+            .where(((cls.model.tenant_id.in_(joined_tenant_ids)) | (cls.model.tenant_id == user_id)) & (cls.model.status == StatusEnum.VALID.value))
+        )
+
+        if keywords:
+            query = query.where(fn.LOWER(cls.model.name).contains(keywords.lower()))
+        if desc:
+            query = query.order_by(cls.model.getter_by(orderby).desc())
+        else:
+            query = query.order_by(cls.model.getter_by(orderby).asc())
+
+        count = query.count()
+
+        if page_number and items_per_page:
+            query = query.paginate(page_number, items_per_page)
+
+        return list(query.dicts()), count

api/db/services/task_service.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import os
 import random
 import xxhash
@@ -28,7 +29,7 @@ from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
 from api.utils import current_timestamp, get_uuid
 from deepdoc.parser.excel_parser import RAGFlowExcelParser
-from rag.settings import SVR_QUEUE_NAME
+from rag.settings import get_svr_queue_name
 from rag.utils.storage_factory import STORAGE_IMPL
 from rag.utils.redis_conn import REDIS_CONN
 from api import settings
@@ -36,6 +37,12 @@ from rag.nlp import search
 
 
 def trim_header_by_lines(text: str, max_length) -> str:
+    # Trim header text to maximum length while preserving line breaks
+    # Args:
+    #     text: Input text to trim
+    #     max_length: Maximum allowed length
+    # Returns:
+    #     Trimmed text
     len_text = len(text)
     if len_text <= max_length:
         return text
@@ -46,11 +53,37 @@ def trim_header_by_lines(text: str, max_length) -> str:
 
 
 class TaskService(CommonService):
+    """Service class for managing document processing tasks.
+    
+    This class extends CommonService to provide specialized functionality for document
+    processing task management, including task creation, progress tracking, and chunk
+    management. It handles various document types (PDF, Excel, etc.) and manages their
+    processing lifecycle.
+    
+    The class implements a robust task queue system with retry mechanisms and progress
+    tracking, supporting both synchronous and asynchronous task execution.
+    
+    Attributes:
+        model: The Task model class for database operations.
+    """
     model = Task
 
     @classmethod
     @DB.connection_context()
     def get_task(cls, task_id):
+        """Retrieve detailed task information by task ID.
+    
+        This method fetches comprehensive task details including associated document,
+        knowledge base, and tenant information. It also handles task retry logic and
+        progress updates.
+    
+        Args:
+            task_id (str): The unique identifier of the task to retrieve.
+    
+        Returns:
+            dict: Task details dictionary containing all task information and related metadata.
+                 Returns None if task is not found or has exceeded retry limit.
+        """
         fields = [
             cls.model.id,
             cls.model.doc_id,
@@ -105,6 +138,18 @@ class TaskService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_tasks(cls, doc_id: str):
+        """Retrieve all tasks associated with a document.
+    
+        This method fetches all processing tasks for a given document, ordered by page
+        number and creation time. It includes task progress and chunk information.
+    
+        Args:
+            doc_id (str): The unique identifier of the document.
+    
+        Returns:
+            list[dict]: List of task dictionaries containing task details.
+                       Returns None if no tasks are found.
+        """
         fields = [
             cls.model.id,
             cls.model.from_page,
@@ -124,11 +169,31 @@ class TaskService(CommonService):
     @classmethod
     @DB.connection_context()
     def update_chunk_ids(cls, id: str, chunk_ids: str):
+        """Update the chunk IDs associated with a task.
+    
+        This method updates the chunk_ids field of a task, which stores the IDs of
+        processed document chunks in a space-separated string format.
+    
+        Args:
+            id (str): The unique identifier of the task.
+            chunk_ids (str): Space-separated string of chunk identifiers.
+        """
         cls.model.update(chunk_ids=chunk_ids).where(cls.model.id == id).execute()
 
     @classmethod
     @DB.connection_context()
     def get_ongoing_doc_name(cls):
+        """Get names of documents that are currently being processed.
+    
+        This method retrieves information about documents that are in the processing state,
+        including their locations and associated IDs. It uses database locking to ensure
+        thread safety when accessing the task information.
+    
+        Returns:
+            list[tuple]: A list of tuples, each containing (parent_id/kb_id, location)
+                        for documents currently being processed. Returns empty list if
+                        no documents are being processed.
+        """
         with DB.lock("get_task", -1):
             docs = (
                 cls.model.select(
@@ -172,6 +237,18 @@ class TaskService(CommonService):
     @classmethod
     @DB.connection_context()
     def do_cancel(cls, id):
+        """Check if a task should be cancelled based on its document status.
+    
+        This method determines whether a task should be cancelled by checking the
+        associated document's run status and progress. A task should be cancelled
+        if its document is marked for cancellation or has negative progress.
+    
+        Args:
+            id (str): The unique identifier of the task to check.
+    
+        Returns:
+            bool: True if the task should be cancelled, False otherwise.
+        """
         task = cls.model.get_by_id(id)
         _, doc = DocumentService.get_by_id(task.doc_id)
         return doc.run == TaskStatus.CANCEL.value or doc.progress < 0
@@ -179,29 +256,79 @@ class TaskService(CommonService):
     @classmethod
     @DB.connection_context()
     def update_progress(cls, id, info):
+        """Update the progress information for a task.
+
+        This method updates both the progress message and completion percentage of a task.
+        It handles platform-specific behavior (macOS vs others) and uses database locking
+        when necessary to ensure thread safety.
+
+        Update Rules:
+            - progress_msg: Always appends the new message to the existing one, and trims the result to max 3000 lines.
+            - progress: Only updates if the current progress is not -1 AND
+                        (the new progress is -1 OR greater than the existing progress),
+                        to avoid overwriting valid progress with invalid or regressive values.
+
+        Args:
+            id (str): The unique identifier of the task to update.
+            info (dict): Dictionary containing progress information with keys:
+                        - progress_msg (str, optional): Progress message to append
+                        - progress (float, optional): Progress percentage (0.0 to 1.0)
+        """
+        task = cls.model.get_by_id(id)
+        if not task:
+            logging.warning("Update_progress error: task not found")
+            return
+
         if os.environ.get("MACOS"):
             if info["progress_msg"]:
-                task = cls.model.get_by_id(id)
                 progress_msg = trim_header_by_lines(task.progress_msg + "\n" + info["progress_msg"], 3000)
                 cls.model.update(progress_msg=progress_msg).where(cls.model.id == id).execute()
             if "progress" in info:
-                cls.model.update(progress=info["progress"]).where(
-                    cls.model.id == id
+                prog = info["progress"]
+                cls.model.update(progress=prog).where(
+                    (cls.model.id == id) &
+                    (
+                        (cls.model.progress != -1) &
+                        ((prog == -1) | (prog > cls.model.progress))
+                    )
                 ).execute()
             return
 
         with DB.lock("update_progress", -1):
             if info["progress_msg"]:
-                task = cls.model.get_by_id(id)
                 progress_msg = trim_header_by_lines(task.progress_msg + "\n" + info["progress_msg"], 3000)
                 cls.model.update(progress_msg=progress_msg).where(cls.model.id == id).execute()
             if "progress" in info:
-                cls.model.update(progress=info["progress"]).where(
-                    cls.model.id == id
+                prog = info["progress"]
+                cls.model.update(progress=prog).where(
+                    (cls.model.id == id) &
+                    (
+                        (cls.model.progress != -1) &
+                        ((prog == -1) | (prog > cls.model.progress))
+                    )
                 ).execute()
 
 
-def queue_tasks(doc: dict, bucket: str, name: str):
+def queue_tasks(doc: dict, bucket: str, name: str, priority: int):
+    """Create and queue document processing tasks.
+    
+    This function creates processing tasks for a document based on its type and configuration.
+    It handles different document types (PDF, Excel, etc.) differently and manages task
+    chunking and configuration. It also implements task reuse optimization by checking
+    for previously completed tasks.
+    
+    Args:
+        doc (dict): Document dictionary containing metadata and configuration.
+        bucket (str): Storage bucket name where the document is stored.
+        name (str): File name of the document.
+        priority (int, optional): Priority level for task queueing (default is 0).
+    
+    Note:
+        - For PDF documents, tasks are created per page range based on configuration
+        - For Excel documents, tasks are created per row range
+        - Task digests are calculated for optimization and reuse
+        - Previous task chunks may be reused if available
+    """
     def new_task():
         return {"id": get_uuid(), "doc_id": doc["id"], "progress": 0.0, "from_page": 0, "to_page": 100000000}
 
@@ -252,6 +379,7 @@ def queue_tasks(doc: dict, bucket: str, name: str):
         task_digest = hasher.hexdigest()
         task["digest"] = task_digest
         task["progress"] = 0.0
+        task["priority"] = priority
 
     prev_tasks = TaskService.get_tasks(doc["id"])
     ck_num = 0
@@ -274,11 +402,31 @@ def queue_tasks(doc: dict, bucket: str, name: str):
     unfinished_task_array = [task for task in parse_task_array if task["progress"] < 1.0]
     for unfinished_task in unfinished_task_array:
         assert REDIS_CONN.queue_product(
-            SVR_QUEUE_NAME, message=unfinished_task
+            get_svr_queue_name(priority), message=unfinished_task
         ), "Can't access Redis. Please check the Redis' status."
 
 
 def reuse_prev_task_chunks(task: dict, prev_tasks: list[dict], chunking_config: dict):
+    """Attempt to reuse chunks from previous tasks for optimization.
+    
+    This function checks if chunks from previously completed tasks can be reused for
+    the current task, which can significantly improve processing efficiency. It matches
+    tasks based on page ranges and configuration digests.
+    
+    Args:
+        task (dict): Current task dictionary to potentially reuse chunks for.
+        prev_tasks (list[dict]): List of previous task dictionaries to check for reuse.
+        chunking_config (dict): Configuration dictionary for chunk processing.
+    
+    Returns:
+        int: Number of chunks successfully reused. Returns 0 if no chunks could be reused.
+    
+    Note:
+        Chunks can only be reused if:
+        - A previous task exists with matching page range and configuration digest
+        - The previous task was completed successfully (progress = 1.0)
+        - The previous task has valid chunk IDs
+    """
     idx = 0
     while idx < len(prev_tasks):
         prev_task = prev_tasks[idx]

api/db/services/user_canvas_version.py

@@ -0,0 +1,46 @@
+from api.db.db_models import UserCanvasVersion, DB
+from api.db.services.common_service import CommonService
+from peewee import DoesNotExist
+
+class UserCanvasVersionService(CommonService):
+    model = UserCanvasVersion
+    
+    
+    @classmethod
+    @DB.connection_context()
+    def list_by_canvas_id(cls, user_canvas_id):
+        try:
+            user_canvas_version = cls.model.select(
+                *[cls.model.id, 
+                cls.model.create_time,
+                cls.model.title,
+                cls.model.create_date, 
+                cls.model.update_date,
+                cls.model.user_canvas_id, 
+                cls.model.update_time]
+            ).where(cls.model.user_canvas_id == user_canvas_id)
+            return user_canvas_version
+        except DoesNotExist:
+            return None
+        except Exception:
+            return None
+    
+    @classmethod
+    @DB.connection_context()
+    def delete_all_versions(cls, user_canvas_id):
+        try:
+            user_canvas_version = cls.model.select().where(cls.model.user_canvas_id == user_canvas_id).order_by(cls.model.create_time.desc())
+            if user_canvas_version.count() > 20:
+                delete_ids = []
+                for i in range(20, user_canvas_version.count()):
+                    delete_ids.append(user_canvas_version[i].id)
+                
+                cls.delete_by_ids(delete_ids)
+            return True
+        except DoesNotExist:
+            return None
+        except Exception:
+            return None
+
+
+

api/db/services/user_service.py

@@ -15,6 +15,7 @@
 #
 import hashlib
 from datetime import datetime
+import logging
 
 import peewee
 from werkzeug.security import generate_password_hash, check_password_hash
@@ -29,11 +30,51 @@ from rag.settings import MINIO
 
 
 class UserService(CommonService):
+    """Service class for managing user-related database operations.
+
+    This class extends CommonService to provide specialized functionality for user management,
+    including authentication, user creation, updates, and deletions.
+
+    Attributes:
+        model: The User model class for database operations.
+    """
     model = User
 
+    @classmethod
+    @DB.connection_context()
+    def query(cls, cols=None, reverse=None, order_by=None, **kwargs):
+        if 'access_token' in kwargs:
+            access_token = kwargs['access_token']
+            
+            # Reject empty, None, or whitespace-only access tokens
+            if not access_token or not str(access_token).strip():
+                logging.warning("UserService.query: Rejecting empty access_token query")
+                return cls.model.select().where(cls.model.id == "INVALID_EMPTY_TOKEN")  # Returns empty result
+            
+            # Reject tokens that are too short (should be UUID, 32+ chars)
+            if len(str(access_token).strip()) < 32:
+                logging.warning(f"UserService.query: Rejecting short access_token query: {len(str(access_token))} chars")
+                return cls.model.select().where(cls.model.id == "INVALID_SHORT_TOKEN")  # Returns empty result
+            
+            # Reject tokens that start with "INVALID_" (from logout)
+            if str(access_token).startswith("INVALID_"):
+                logging.warning("UserService.query: Rejecting invalidated access_token")
+                return cls.model.select().where(cls.model.id == "INVALID_LOGOUT_TOKEN")  # Returns empty result
+        
+        # Call parent query method for valid requests
+        return super().query(cols=cols, reverse=reverse, order_by=order_by, **kwargs)
+
     @classmethod
     @DB.connection_context()
     def filter_by_id(cls, user_id):
+        """Retrieve a user by their ID.
+
+        Args:
+            user_id: The unique identifier of the user.
+
+        Returns:
+            User object if found, None otherwise.
+        """
         try:
             user = cls.model.select().where(cls.model.id == user_id).get()
             return user
@@ -43,6 +84,15 @@ class UserService(CommonService):
     @classmethod
     @DB.connection_context()
     def query_user(cls, email, password):
+        """Authenticate a user with email and password.
+
+        Args:
+            email: User's email address.
+            password: User's password in plain text.
+
+        Returns:
+            User object if authentication successful, None otherwise.
+        """
         user = cls.model.select().where((cls.model.email == email),
                                         (cls.model.status == StatusEnum.VALID.value)).first()
         if user and check_password_hash(str(user.password), password):
@@ -85,6 +135,14 @@ class UserService(CommonService):
 
 
 class TenantService(CommonService):
+    """Service class for managing tenant-related database operations.
+
+    This class extends CommonService to provide functionality for tenant management,
+    including tenant information retrieval and credit management.
+
+    Attributes:
+        model: The Tenant model class for database operations.
+    """
     model = Tenant
 
     @classmethod
@@ -136,8 +194,25 @@ class TenantService(CommonService):
 
 
 class UserTenantService(CommonService):
+    """Service class for managing user-tenant relationship operations.
+
+    This class extends CommonService to handle the many-to-many relationship
+    between users and tenants, managing user roles and tenant memberships.
+
+    Attributes:
+        model: The UserTenant model class for database operations.
+    """
     model = UserTenant
 
+    @classmethod
+    @DB.connection_context()
+    def filter_by_id(cls, user_tenant_id):
+        try:
+            user_tenant = cls.model.select().where((cls.model.id == user_tenant_id) & (cls.model.status == StatusEnum.VALID.value)).get()
+            return user_tenant
+        except peewee.DoesNotExist:
+            return None
+
     @classmethod
     @DB.connection_context()
     def save(cls, **kwargs):
@@ -150,6 +225,7 @@ class UserTenantService(CommonService):
     @DB.connection_context()
     def get_by_tenant_id(cls, tenant_id):
         fields = [
+            cls.model.id,
             cls.model.user_id,
             cls.model.status,
             cls.model.role,
@@ -181,3 +257,21 @@ class UserTenantService(CommonService):
         return list(cls.model.select(*fields)
                     .join(User, on=((cls.model.tenant_id == User.id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value)))
                     .where(cls.model.status == StatusEnum.VALID.value).dicts())
+
+    @classmethod
+    @DB.connection_context()
+    def get_num_members(cls, user_id: str):
+        cnt_members = cls.model.select(peewee.fn.COUNT(cls.model.id)).where(cls.model.tenant_id == user_id).scalar()
+        return cnt_members
+
+    @classmethod
+    @DB.connection_context()
+    def filter_by_tenant_and_user_id(cls, tenant_id, user_id):
+        try:
+            user_tenant = cls.model.select().where(
+                (cls.model.tenant_id == tenant_id) & (cls.model.status == StatusEnum.VALID.value) &
+                (cls.model.user_id == user_id)
+            ).first()
+            return user_tenant
+        except peewee.DoesNotExist:
+            return None

api/ragflow_server.py

@@ -18,8 +18,9 @@
 # from beartype.claw import beartype_all  # <-- you didn't sign up for this
 # beartype_all(conf=BeartypeConf(violation_type=UserWarning))    # <-- emit warnings from all code
 
-from api.utils.log_utils import initRootLogger
-initRootLogger("ragflow_server")
+from api.utils.log_utils import init_root_logger
+from plugin import GlobalPluginManager
+init_root_logger("ragflow_server")
 
 import logging
 import os
@@ -27,7 +28,8 @@ import signal
 import sys
 import time
 import traceback
-from concurrent.futures import ThreadPoolExecutor
+import threading
+import uuid
 
 from werkzeug.serving import run_simple
 from api import settings
@@ -41,16 +43,32 @@ from api.db.init_data import init_web_data
 from api.versions import get_ragflow_version
 from api.utils import show_configs
 from rag.settings import print_rag_settings
+from rag.utils.redis_conn import RedisDistributedLock
 
+stop_event = threading.Event()
+
+RAGFLOW_DEBUGPY_LISTEN = int(os.environ.get('RAGFLOW_DEBUGPY_LISTEN', "0"))
 
 def update_progress():
-    while True:
-        time.sleep(6)
+    lock_value = str(uuid.uuid4())
+    redis_lock = RedisDistributedLock("update_progress", lock_value=lock_value, timeout=60)
+    logging.info(f"update_progress lock_value: {lock_value}")
+    while not stop_event.is_set():
         try:
-            DocumentService.update_progress()
+            if redis_lock.acquire():
+                DocumentService.update_progress()
+                redis_lock.release()
+            stop_event.wait(6)
         except Exception:
             logging.exception("update_progress exception")
+        finally:
+            redis_lock.release()
 
+def signal_handler(sig, frame):
+    logging.info("Received interrupt signal, shutting down...")
+    stop_event.set()
+    time.sleep(1)
+    sys.exit(0)
 
 if __name__ == '__main__':
     logging.info(r"""
@@ -71,6 +89,11 @@ if __name__ == '__main__':
     settings.init_settings()
     print_rag_settings()
 
+    if RAGFLOW_DEBUGPY_LISTEN > 0:
+        logging.info(f"debugpy listen on {RAGFLOW_DEBUGPY_LISTEN}")
+        import debugpy
+        debugpy.listen(("0.0.0.0", RAGFLOW_DEBUGPY_LISTEN))
+
     # init db
     init_web_db()
     init_web_data()
@@ -96,8 +119,21 @@ if __name__ == '__main__':
     RuntimeConfig.init_env()
     RuntimeConfig.init_config(JOB_SERVER_HOST=settings.HOST_IP, HTTP_PORT=settings.HOST_PORT)
 
-    thread = ThreadPoolExecutor(max_workers=1)
-    thread.submit(update_progress)
+    GlobalPluginManager.load_plugins()
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    def delayed_start_update_progress():
+        logging.info("Starting update_progress thread (delayed)")
+        t = threading.Thread(target=update_progress, daemon=True)
+        t.start()
+
+    if RuntimeConfig.DEBUG:
+        if os.environ.get("WERKZEUG_RUN_MAIN") == "true":
+            threading.Timer(1.0, delayed_start_update_progress).start()
+    else:
+        threading.Timer(1.0, delayed_start_update_progress).start()
 
     # start http server
     try:
@@ -112,4 +148,6 @@ if __name__ == '__main__':
         )
     except Exception:
         traceback.print_exc()
+        stop_event.set()
+        time.sleep(1)
         os.kill(os.getpid(), signal.SIGKILL)

api/settings.py

@@ -13,19 +13,23 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import json
 import os
+import secrets
 from datetime import date
-from enum import IntEnum, Enum
-import rag.utils.es_conn
-import rag.utils.infinity_conn
+from enum import Enum, IntEnum
 
 import rag.utils
-from rag.nlp import search
-from graphrag import search as kg_search
-from api.utils import get_base_config, decrypt_database_config
+import rag.utils.es_conn
+import rag.utils.infinity_conn
+import rag.utils.opensearch_coon
 from api.constants import RAG_FLOW_SERVICE_NAME
+from api.utils import decrypt_database_config, get_base_config
+from api.utils.file_utils import get_project_base_directory
+from graphrag import search as kg_search
+from rag.nlp import search
 
-LIGHTEN = int(os.environ.get('LIGHTEN', "0"))
+LIGHTEN = int(os.environ.get("LIGHTEN", "0"))
 
 LLM = None
 LLM_FACTORY = None
@@ -40,8 +44,9 @@ PARSERS = None
 HOST_IP = None
 HOST_PORT = None
 SECRET_KEY = None
+FACTORY_LLM_INFOS = None
 
-DATABASE_TYPE = os.getenv("DB_TYPE", 'mysql')
+DATABASE_TYPE = os.getenv("DB_TYPE", "mysql")
 DATABASE = decrypt_database_config(name=DATABASE_TYPE)
 
 # authentication
@@ -52,128 +57,124 @@ CLIENT_AUTHENTICATION = None
 HTTP_APP_KEY = None
 GITHUB_OAUTH = None
 FEISHU_OAUTH = None
-
+OAUTH_CONFIG = None
 DOC_ENGINE = None
 docStoreConn = None
 
 retrievaler = None
 kg_retrievaler = None
 
+# user registration switch
+REGISTER_ENABLED = 1
+
+
+# sandbox-executor-manager
+SANDBOX_ENABLED = 0
+SANDBOX_HOST = None
+
+BUILTIN_EMBEDDING_MODELS = ["BAAI/bge-large-zh-v1.5@BAAI", "maidalun1020/bce-embedding-base_v1@Youdao"]
+
+def get_or_create_secret_key():
+    secret_key = os.environ.get("RAGFLOW_SECRET_KEY")
+    if secret_key and len(secret_key) >= 32:
+        return secret_key
+    
+    # Check if there's a configured secret key
+    configured_key = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("secret_key")
+    if configured_key and configured_key != str(date.today()) and len(configured_key) >= 32:
+        return configured_key
+    
+    # Generate a new secure key and warn about it
+    import logging
+    new_key = secrets.token_hex(32)
+    logging.warning(
+        "SECURITY WARNING: Using auto-generated SECRET_KEY. "
+        f"Generated key: {new_key}"
+    )
+    return new_key
+
 
 def init_settings():
-    global LLM, LLM_FACTORY, LLM_BASE_URL, LIGHTEN, DATABASE_TYPE, DATABASE
-    LIGHTEN = int(os.environ.get('LIGHTEN', "0"))
-    DATABASE_TYPE = os.getenv("DB_TYPE", 'mysql')
+    global LLM, LLM_FACTORY, LLM_BASE_URL, LIGHTEN, DATABASE_TYPE, DATABASE, FACTORY_LLM_INFOS, REGISTER_ENABLED
+    LIGHTEN = int(os.environ.get("LIGHTEN", "0"))
+    DATABASE_TYPE = os.getenv("DB_TYPE", "mysql")
     DATABASE = decrypt_database_config(name=DATABASE_TYPE)
     LLM = get_base_config("user_default_llm", {})
-    LLM_FACTORY = LLM.get("factory", "Tongyi-Qianwen")
+    LLM_DEFAULT_MODELS = LLM.get("default_models", {})
+    LLM_FACTORY = LLM.get("factory")
     LLM_BASE_URL = LLM.get("base_url")
+    try:
+        REGISTER_ENABLED = int(os.environ.get("REGISTER_ENABLED", "1"))
+    except Exception:
+        pass
+
+    try:
+        with open(os.path.join(get_project_base_directory(), "conf", "llm_factories.json"), "r") as f:
+            FACTORY_LLM_INFOS = json.load(f)["factory_llm_infos"]
+    except Exception:
+        FACTORY_LLM_INFOS = []
 
     global CHAT_MDL, EMBEDDING_MDL, RERANK_MDL, ASR_MDL, IMAGE2TEXT_MDL
     if not LIGHTEN:
-        default_llm = {
-            "Tongyi-Qianwen": {
-                "chat_model": "qwen-plus",
-                "embedding_model": "text-embedding-v2",
-                "image2text_model": "qwen-vl-max",
-                "asr_model": "paraformer-realtime-8k-v1",
-            },
-            "OpenAI": {
-                "chat_model": "gpt-3.5-turbo",
-                "embedding_model": "text-embedding-ada-002",
-                "image2text_model": "gpt-4-vision-preview",
-                "asr_model": "whisper-1",
-            },
-            "Azure-OpenAI": {
-                "chat_model": "gpt-35-turbo",
-                "embedding_model": "text-embedding-ada-002",
-                "image2text_model": "gpt-4-vision-preview",
-                "asr_model": "whisper-1",
-            },
-            "ZHIPU-AI": {
-                "chat_model": "glm-3-turbo",
-                "embedding_model": "embedding-2",
-                "image2text_model": "glm-4v",
-                "asr_model": "",
-            },
-            "Ollama": {
-                "chat_model": "qwen-14B-chat",
-                "embedding_model": "flag-embedding",
-                "image2text_model": "",
-                "asr_model": "",
-            },
-            "Moonshot": {
-                "chat_model": "moonshot-v1-8k",
-                "embedding_model": "",
-                "image2text_model": "",
-                "asr_model": "",
-            },
-            "DeepSeek": {
-                "chat_model": "deepseek-chat",
-                "embedding_model": "",
-                "image2text_model": "",
-                "asr_model": "",
-            },
-            "VolcEngine": {
-                "chat_model": "",
-                "embedding_model": "",
-                "image2text_model": "",
-                "asr_model": "",
-            },
-            "BAAI": {
-                "chat_model": "",
-                "embedding_model": "BAAI/bge-large-zh-v1.5",
-                "image2text_model": "",
-                "asr_model": "",
-                "rerank_model": "BAAI/bge-reranker-v2-m3",
-            }
-        }
+        EMBEDDING_MDL = BUILTIN_EMBEDDING_MODELS[0]
 
-        if LLM_FACTORY:
-            CHAT_MDL = default_llm[LLM_FACTORY]["chat_model"] + f"@{LLM_FACTORY}"
-            ASR_MDL = default_llm[LLM_FACTORY]["asr_model"] + f"@{LLM_FACTORY}"
-            IMAGE2TEXT_MDL = default_llm[LLM_FACTORY]["image2text_model"] + f"@{LLM_FACTORY}"
-        EMBEDDING_MDL = default_llm["BAAI"]["embedding_model"] + "@BAAI"
-        RERANK_MDL = default_llm["BAAI"]["rerank_model"] + "@BAAI"
+    if LLM_DEFAULT_MODELS:
+        CHAT_MDL = LLM_DEFAULT_MODELS.get("chat_model", CHAT_MDL)
+        EMBEDDING_MDL = LLM_DEFAULT_MODELS.get("embedding_model", EMBEDDING_MDL)
+        RERANK_MDL = LLM_DEFAULT_MODELS.get("rerank_model", RERANK_MDL)
+        ASR_MDL = LLM_DEFAULT_MODELS.get("asr_model", ASR_MDL)
+        IMAGE2TEXT_MDL = LLM_DEFAULT_MODELS.get("image2text_model", IMAGE2TEXT_MDL)
+
+        # factory can be specified in the config name with "@". LLM_FACTORY will be used if not specified
+        CHAT_MDL = CHAT_MDL + (f"@{LLM_FACTORY}" if "@" not in CHAT_MDL and CHAT_MDL != "" else "")
+        EMBEDDING_MDL = EMBEDDING_MDL + (f"@{LLM_FACTORY}" if "@" not in EMBEDDING_MDL and EMBEDDING_MDL != "" else "")
+        RERANK_MDL = RERANK_MDL + (f"@{LLM_FACTORY}" if "@" not in RERANK_MDL and RERANK_MDL != "" else "")
+        ASR_MDL = ASR_MDL + (f"@{LLM_FACTORY}" if "@" not in ASR_MDL and ASR_MDL != "" else "")
+        IMAGE2TEXT_MDL = IMAGE2TEXT_MDL + (f"@{LLM_FACTORY}" if "@" not in IMAGE2TEXT_MDL and IMAGE2TEXT_MDL != "" else "")
 
     global API_KEY, PARSERS, HOST_IP, HOST_PORT, SECRET_KEY
-    API_KEY = LLM.get("api_key", "")
+    API_KEY = LLM.get("api_key")
     PARSERS = LLM.get(
-        "parsers",
-        "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,knowledge_graph:Knowledge Graph,email:Email,tag:Tag")
+        "parsers", "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email,tag:Tag"
+    )
 
     HOST_IP = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("host", "127.0.0.1")
     HOST_PORT = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("http_port")
 
-    SECRET_KEY = get_base_config(
-        RAG_FLOW_SERVICE_NAME,
-        {}).get("secret_key", str(date.today()))
+    SECRET_KEY = get_or_create_secret_key()
 
-    global AUTHENTICATION_CONF, CLIENT_AUTHENTICATION, HTTP_APP_KEY, GITHUB_OAUTH, FEISHU_OAUTH
+    global AUTHENTICATION_CONF, CLIENT_AUTHENTICATION, HTTP_APP_KEY, GITHUB_OAUTH, FEISHU_OAUTH, OAUTH_CONFIG
     # authentication
     AUTHENTICATION_CONF = get_base_config("authentication", {})
 
     # client
-    CLIENT_AUTHENTICATION = AUTHENTICATION_CONF.get(
-        "client", {}).get(
-        "switch", False)
+    CLIENT_AUTHENTICATION = AUTHENTICATION_CONF.get("client", {}).get("switch", False)
     HTTP_APP_KEY = AUTHENTICATION_CONF.get("client", {}).get("http_app_key")
     GITHUB_OAUTH = get_base_config("oauth", {}).get("github")
     FEISHU_OAUTH = get_base_config("oauth", {}).get("feishu")
 
+    OAUTH_CONFIG = get_base_config("oauth", {})
+
     global DOC_ENGINE, docStoreConn, retrievaler, kg_retrievaler
-    DOC_ENGINE = os.environ.get('DOC_ENGINE', "elasticsearch")
+    DOC_ENGINE = os.environ.get("DOC_ENGINE", "elasticsearch")
+    # DOC_ENGINE = os.environ.get('DOC_ENGINE', "opensearch")
     lower_case_doc_engine = DOC_ENGINE.lower()
     if lower_case_doc_engine == "elasticsearch":
         docStoreConn = rag.utils.es_conn.ESConnection()
     elif lower_case_doc_engine == "infinity":
         docStoreConn = rag.utils.infinity_conn.InfinityConnection()
+    elif lower_case_doc_engine == "opensearch":
+        docStoreConn = rag.utils.opensearch_coon.OSConnection()
     else:
         raise Exception(f"Not supported doc engine: {DOC_ENGINE}")
 
     retrievaler = search.Dealer(docStoreConn)
     kg_retrievaler = kg_search.KGSearch(docStoreConn)
 
+    if int(os.environ.get("SANDBOX_ENABLED", "0")):
+        global SANDBOX_HOST
+        SANDBOX_HOST = os.environ.get("SANDBOX_HOST", "sandbox-executor-manager")
+
 
 class CustomEnum(Enum):
     @classmethod

api/utils/__init__.py

@@ -70,6 +70,12 @@ def show_configs():
             if "password" in v:
                 v = copy.deepcopy(v)
                 v["password"] = "*" * 8
+            if "access_key" in v:
+                v = copy.deepcopy(v)
+                v["access_key"] = "*" * 8
+            if "secret_key" in v:
+                v = copy.deepcopy(v)
+                v["secret_key"] = "*" * 8
         msg += f"\n\t{k}: {v}"
     logging.info(msg)
 
@@ -351,6 +357,26 @@ def decrypt(line):
         line), "Fail to decrypt password!").decode('utf-8')
 
 
+def decrypt2(crypt_text):
+    from base64 import b64decode, b16decode
+    from Crypto.Cipher import PKCS1_v1_5 as Cipher_PKCS1_v1_5
+    from Crypto.PublicKey import RSA
+    decode_data = b64decode(crypt_text)
+    if len(decode_data) == 127:
+        hex_fixed = '00' + decode_data.hex()
+        decode_data = b16decode(hex_fixed.upper())
+
+    file_path = os.path.join(
+        file_utils.get_project_base_directory(),
+        "conf",
+        "private.pem")
+    pem = open(file_path).read()
+    rsa_key = RSA.importKey(pem, "Welcome")
+    cipher = Cipher_PKCS1_v1_5.new(rsa_key)
+    decrypt_text = cipher.decrypt(decode_data, None)
+    return (b64decode(decrypt_text)).decode()
+
+
 def download_img(url):
     if not url:
         return ""

api/utils/api_utils.py

@@ -13,12 +13,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import logging
 import functools
 import json
+import logging
 import random
 import time
 from base64 import b64encode
+from copy import deepcopy
 from functools import wraps
 from hmac import HMAC
 from io import BytesIO
@@ -27,59 +28,62 @@ from uuid import uuid1
 
 import requests
 from flask import (
-    Response, jsonify, send_file, make_response,
+    Response,
+    jsonify,
+    make_response,
+    send_file,
+)
+from flask import (
     request as flask_request,
 )
 from itsdangerous import URLSafeTimedSerializer
+from peewee import OperationalError
 from werkzeug.http import HTTP_STATUS_CODES
 
-from api.db.db_models import APIToken
 from api import settings
+from api.constants import REQUEST_MAX_WAIT_SEC, REQUEST_WAIT_SEC
+from api.db.db_models import APIToken
+from api.db.services.llm_service import LLMService, TenantLLMService
+from api.utils import CustomJSONEncoder, get_uuid, json_dumps
 
-from api.utils import CustomJSONEncoder, get_uuid
-from api.utils import json_dumps
-from api.constants import REQUEST_WAIT_SEC, REQUEST_MAX_WAIT_SEC
-
-requests.models.complexjson.dumps = functools.partial(
-    json.dumps, cls=CustomJSONEncoder)
+requests.models.complexjson.dumps = functools.partial(json.dumps, cls=CustomJSONEncoder)
 
 
 def request(**kwargs):
     sess = requests.Session()
-    stream = kwargs.pop('stream', sess.stream)
-    timeout = kwargs.pop('timeout', None)
-    kwargs['headers'] = {
-        k.replace(
-            '_',
-            '-').upper(): v for k,
-                                v in kwargs.get(
-            'headers',
-            {}).items()}
+    stream = kwargs.pop("stream", sess.stream)
+    timeout = kwargs.pop("timeout", None)
+    kwargs["headers"] = {k.replace("_", "-").upper(): v for k, v in kwargs.get("headers", {}).items()}
     prepped = requests.Request(**kwargs).prepare()
 
     if settings.CLIENT_AUTHENTICATION and settings.HTTP_APP_KEY and settings.SECRET_KEY:
         timestamp = str(round(time() * 1000))
         nonce = str(uuid1())
-        signature = b64encode(HMAC(settings.SECRET_KEY.encode('ascii'), b'\n'.join([
-            timestamp.encode('ascii'),
-            nonce.encode('ascii'),
-            settings.HTTP_APP_KEY.encode('ascii'),
-            prepped.path_url.encode('ascii'),
-            prepped.body if kwargs.get('json') else b'',
-            urlencode(
-                sorted(
-                    kwargs['data'].items()),
-                quote_via=quote,
-                safe='-._~').encode('ascii')
-            if kwargs.get('data') and isinstance(kwargs['data'], dict) else b'',
-        ]), 'sha1').digest()).decode('ascii')
+        signature = b64encode(
+            HMAC(
+                settings.SECRET_KEY.encode("ascii"),
+                b"\n".join(
+                    [
+                        timestamp.encode("ascii"),
+                        nonce.encode("ascii"),
+                        settings.HTTP_APP_KEY.encode("ascii"),
+                        prepped.path_url.encode("ascii"),
+                        prepped.body if kwargs.get("json") else b"",
+                        urlencode(sorted(kwargs["data"].items()), quote_via=quote, safe="-._~").encode("ascii") if kwargs.get("data") and isinstance(kwargs["data"], dict) else b"",
+                    ]
+                ),
+                "sha1",
+            ).digest()
+        ).decode("ascii")
 
-        prepped.headers.update({
-            'TIMESTAMP': timestamp,
-            'NONCE': nonce,
-            'APP-KEY': settings.HTTP_APP_KEY,
-            'SIGNATURE': signature,
-        })
+        prepped.headers.update(
+            {
+                "TIMESTAMP": timestamp,
+                "NONCE": nonce,
+                "APP-KEY": settings.HTTP_APP_KEY,
+                "SIGNATURE": signature,
+            }
+        )
 
     return sess.send(prepped, stream=stream, timeout=timeout)
 
@@ -87,7 +91,7 @@ def request(**kwargs):
 def get_exponential_backoff_interval(retries, full_jitter=False):
     """Calculate the exponential backoff wait time."""
     # Will be zero if factor equals 0
-    countdown = min(REQUEST_MAX_WAIT_SEC, REQUEST_WAIT_SEC * (2 ** retries))
+    countdown = min(REQUEST_MAX_WAIT_SEC, REQUEST_WAIT_SEC * (2**retries))
     # Full jitter according to
     # https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
     if full_jitter:
@@ -96,12 +100,9 @@ def get_exponential_backoff_interval(retries, full_jitter=False):
     return max(0, countdown)
 
 
-def get_data_error_result(code=settings.RetCode.DATA_ERROR,
-                          message='Sorry! Data missing!'):
+def get_data_error_result(code=settings.RetCode.DATA_ERROR, message="Sorry! Data missing!"):
     logging.exception(Exception(message))
-    result_dict = {
-        "code": code,
-        "message": message}
+    result_dict = {"code": code, "message": message}
     response = {}
     for key, value in result_dict.items():
         if value is None and key != "code":
@@ -119,23 +120,27 @@ def server_error_response(e):
     except BaseException:
         pass
     if len(e.args) > 1:
-        return get_json_result(
-            code=settings.RetCode.EXCEPTION_ERROR, message=repr(e.args[0]), data=e.args[1])
+        return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message=repr(e.args[0]), data=e.args[1])
     if repr(e).find("index_not_found_exception") >= 0:
-        return get_json_result(code=settings.RetCode.EXCEPTION_ERROR,
-                               message="No chunk found, please upload file and parse it.")
+        return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message="No chunk found, please upload file and parse it.")
 
     return get_json_result(code=settings.RetCode.EXCEPTION_ERROR, message=repr(e))
 
 
 def error_response(response_code, message=None):
     if message is None:
-        message = HTTP_STATUS_CODES.get(response_code, 'Unknown Error')
+        message = HTTP_STATUS_CODES.get(response_code, "Unknown Error")
 
-    return Response(json.dumps({
-        'message': message,
-        'code': response_code,
-    }), status=response_code, mimetype='application/json')
+    return Response(
+        json.dumps(
+            {
+                "message": message,
+                "code": response_code,
+            }
+        ),
+        status=response_code,
+        mimetype="application/json",
+    )
 
 
 def validate_request(*args, **kwargs):
@@ -160,13 +165,10 @@ def validate_request(*args, **kwargs):
             if no_arguments or error_arguments:
                 error_string = ""
                 if no_arguments:
-                    error_string += "required argument are missing: {}; ".format(
-                        ",".join(no_arguments))
+                    error_string += "required argument are missing: {}; ".format(",".join(no_arguments))
                 if error_arguments:
-                    error_string += "required argument values: {}".format(
-                        ",".join(["{}={}".format(a[0], a[1]) for a in error_arguments]))
-                return get_json_result(
-                    code=settings.RetCode.ARGUMENT_ERROR, message=error_string)
+                    error_string += "required argument values: {}".format(",".join(["{}={}".format(a[0], a[1]) for a in error_arguments]))
+                return get_json_result(code=settings.RetCode.ARGUMENT_ERROR, message=error_string)
             return func(*_args, **_kwargs)
 
         return decorated_function
@@ -180,8 +182,7 @@ def not_allowed_parameters(*params):
             input_arguments = flask_request.json or flask_request.form.to_dict()
             for param in params:
                 if param in input_arguments:
-                    return get_json_result(
-                        code=settings.RetCode.ARGUMENT_ERROR, message=f"Parameter {param} isn't allowed")
+                    return get_json_result(code=settings.RetCode.ARGUMENT_ERROR, message=f"Parameter {param} isn't allowed")
             return f(*args, **kwargs)
 
         return wrapper
@@ -190,14 +191,14 @@ def not_allowed_parameters(*params):
 
 
 def is_localhost(ip):
-    return ip in {'127.0.0.1', '::1', '[::1]', 'localhost'}
+    return ip in {"127.0.0.1", "::1", "[::1]", "localhost"}
 
 
 def send_file_in_mem(data, filename):
     if not isinstance(data, (str, bytes)):
         data = json_dumps(data)
     if isinstance(data, str):
-        data = data.encode('utf-8')
+        data = data.encode("utf-8")
 
     f = BytesIO()
     f.write(data)
@@ -206,7 +207,7 @@ def send_file_in_mem(data, filename):
     return send_file(f, as_attachment=True, attachment_filename=filename)
 
 
-def get_json_result(code=settings.RetCode.SUCCESS, message='success', data=None):
+def get_json_result(code=settings.RetCode.SUCCESS, message="success", data=None):
     response = {"code": code, "message": message, "data": data}
     return jsonify(response)
 
@@ -214,27 +215,24 @@ def get_json_result(code=settings.RetCode.SUCCESS, message='success', data=None)
 def apikey_required(func):
     @wraps(func)
     def decorated_function(*args, **kwargs):
-        token = flask_request.headers.get('Authorization').split()[1]
+        token = flask_request.headers.get("Authorization").split()[1]
         objs = APIToken.query(token=token)
         if not objs:
-            return build_error_result(
-                message='API-KEY is invalid!', code=settings.RetCode.FORBIDDEN
-            )
-        kwargs['tenant_id'] = objs[0].tenant_id
+            return build_error_result(message="API-KEY is invalid!", code=settings.RetCode.FORBIDDEN)
+        kwargs["tenant_id"] = objs[0].tenant_id
         return func(*args, **kwargs)
 
     return decorated_function
 
 
-def build_error_result(code=settings.RetCode.FORBIDDEN, message='success'):
+def build_error_result(code=settings.RetCode.FORBIDDEN, message="success"):
     response = {"code": code, "message": message}
     response = jsonify(response)
     response.status_code = code
     return response
 
 
-def construct_response(code=settings.RetCode.SUCCESS,
-                       message='success', data=None, auth=None):
+def construct_response(code=settings.RetCode.SUCCESS, message="success", data=None, auth=None):
     result_dict = {"code": code, "message": message, "data": data}
     response_dict = {}
     for key, value in result_dict.items():
@@ -253,7 +251,7 @@ def construct_response(code=settings.RetCode.SUCCESS,
     return response
 
 
-def construct_result(code=settings.RetCode.DATA_ERROR, message='data is missing'):
+def construct_result(code=settings.RetCode.DATA_ERROR, message="data is missing"):
     result_dict = {"code": code, "message": message}
     response = {}
     for key, value in result_dict.items():
@@ -264,7 +262,7 @@ def construct_result(code=settings.RetCode.DATA_ERROR, message='data is missing'
     return jsonify(response)
 
 
-def construct_json_result(code=settings.RetCode.SUCCESS, message='success', data=None):
+def construct_json_result(code=settings.RetCode.SUCCESS, message="success", data=None):
     if data is None:
         return jsonify({"code": code, "message": message})
     else:
@@ -286,7 +284,7 @@ def construct_error_response(e):
 def token_required(func):
     @wraps(func)
     def decorated_function(*args, **kwargs):
-        authorization_str = flask_request.headers.get('Authorization')
+        authorization_str = flask_request.headers.get("Authorization")
         if not authorization_str:
             return get_json_result(data=False, message="`Authorization` can't be empty")
         authorization_list = authorization_str.split()
@@ -295,11 +293,8 @@ def token_required(func):
         token = authorization_list[1]
         objs = APIToken.query(token=token)
         if not objs:
-            return get_json_result(
-                data=False, message='Authentication error: API key is invalid!',
-                code=settings.RetCode.AUTHENTICATION_ERROR
-            )
-        kwargs['tenant_id'] = objs[0].tenant_id
+            return get_json_result(data=False, message="Authentication error: API key is invalid!", code=settings.RetCode.AUTHENTICATION_ERROR)
+        kwargs["tenant_id"] = objs[0].tenant_id
         return func(*args, **kwargs)
 
     return decorated_function
@@ -316,11 +311,11 @@ def get_result(code=settings.RetCode.SUCCESS, message="", data=None):
     return jsonify(response)
 
 
-def get_error_data_result(message='Sorry! Data missing!', code=settings.RetCode.DATA_ERROR,
-                          ):
-    result_dict = {
-        "code": code,
-        "message": message}
+def get_error_data_result(
+    message="Sorry! Data missing!",
+    code=settings.RetCode.DATA_ERROR,
+):
+    result_dict = {"code": code, "message": message}
     response = {}
     for key, value in result_dict.items():
         if value is None and key != "code":
@@ -330,23 +325,21 @@ def get_error_data_result(message='Sorry! Data missing!', code=settings.RetCode.
     return jsonify(response)
 
 
-def generate_confirmation_token(tenent_id):
-    serializer = URLSafeTimedSerializer(tenent_id)
-    return "ragflow-" + serializer.dumps(get_uuid(), salt=tenent_id)[2:34]
+def get_error_argument_result(message="Invalid arguments"):
+    return get_result(code=settings.RetCode.ARGUMENT_ERROR, message=message)
 
 
-def valid(permission, valid_permission, language, valid_language, chunk_method, valid_chunk_method):
-    if valid_parameter(permission, valid_permission):
-        return valid_parameter(permission, valid_permission)
-    if valid_parameter(language, valid_language):
-        return valid_parameter(language, valid_language)
-    if valid_parameter(chunk_method, valid_chunk_method):
-        return valid_parameter(chunk_method, valid_chunk_method)
+def get_error_permission_result(message="Permission error"):
+    return get_result(code=settings.RetCode.PERMISSION_ERROR, message=message)
 
 
-def valid_parameter(parameter, valid_values):
-    if parameter and parameter not in valid_values:
-        return get_error_data_result(f"'{parameter}' is not in {valid_values}")
+def get_error_operating_result(message="Operating error"):
+    return get_result(code=settings.RetCode.OPERATING_ERROR, message=message)
+
+
+def generate_confirmation_token(tenant_id):
+    serializer = URLSafeTimedSerializer(tenant_id)
+    return "ragflow-" + serializer.dumps(get_uuid(), salt=tenant_id)[2:34]
 
 
 def get_parser_config(chunk_method, parser_config):
@@ -355,8 +348,7 @@ def get_parser_config(chunk_method, parser_config):
     if not chunk_method:
         chunk_method = "naive"
     key_mapping = {
-        "naive": {"chunk_token_num": 128, "delimiter": "\\n!?;。；！？", "html4excel": False, "layout_recognize": "DeepDOC",
-                  "raptor": {"use_raptor": False}},
+        "naive": {"chunk_token_num": 128, "delimiter": r"\n", "html4excel": False, "layout_recognize": "DeepDOC", "raptor": {"use_raptor": False}},
         "qa": {"raptor": {"use_raptor": False}},
         "tag": None,
         "resume": None,
@@ -367,9 +359,202 @@ def get_parser_config(chunk_method, parser_config):
         "laws": {"raptor": {"use_raptor": False}},
         "presentation": {"raptor": {"use_raptor": False}},
         "one": None,
-        "knowledge_graph": {"chunk_token_num": 8192, "delimiter": "\\n!?;。；！？",
-                            "entity_types": ["organization", "person", "location", "event", "time"]},
+        "knowledge_graph": {"chunk_token_num": 8192, "delimiter": r"\n", "entity_types": ["organization", "person", "location", "event", "time"]},
         "email": None,
-        "picture": None}
+        "picture": None,
+    }
     parser_config = key_mapping[chunk_method]
     return parser_config
+
+
+def get_data_openai(
+    id=None,
+    created=None,
+    model=None,
+    prompt_tokens=0,
+    completion_tokens=0,
+    content=None,
+    finish_reason=None,
+    object="chat.completion",
+    param=None,
+):
+    total_tokens = prompt_tokens + completion_tokens
+    return {
+        "id": f"{id}",
+        "object": object,
+        "created": int(time.time()) if created else None,
+        "model": model,
+        "param": param,
+        "usage": {
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": total_tokens,
+            "completion_tokens_details": {"reasoning_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0},
+        },
+        "choices": [{"message": {"role": "assistant", "content": content}, "logprobs": None, "finish_reason": finish_reason, "index": 0}],
+    }
+
+
+def check_duplicate_ids(ids, id_type="item"):
+    """
+    Check for duplicate IDs in a list and return unique IDs and error messages.
+
+    Args:
+        ids (list): List of IDs to check for duplicates
+        id_type (str): Type of ID for error messages (e.g., 'document', 'dataset', 'chunk')
+
+    Returns:
+        tuple: (unique_ids, error_messages)
+            - unique_ids (list): List of unique IDs
+            - error_messages (list): List of error messages for duplicate IDs
+    """
+    id_count = {}
+    duplicate_messages = []
+
+    # Count occurrences of each ID
+    for id_value in ids:
+        id_count[id_value] = id_count.get(id_value, 0) + 1
+
+    # Check for duplicates
+    for id_value, count in id_count.items():
+        if count > 1:
+            duplicate_messages.append(f"Duplicate {id_type} ids: {id_value}")
+
+    # Return unique IDs and error messages
+    return list(set(ids)), duplicate_messages
+
+
+def verify_embedding_availability(embd_id: str, tenant_id: str) -> tuple[bool, Response | None]:
+    """
+    Verifies availability of an embedding model for a specific tenant.
+
+    Performs comprehensive verification through:
+    1. Identifier Parsing: Decomposes embd_id into name and factory components
+    2. System Verification: Checks model registration in LLMService
+    3. Tenant Authorization: Validates tenant-specific model assignments
+    4. Built-in Model Check: Confirms inclusion in predefined system models
+
+    Args:
+        embd_id (str): Unique identifier for the embedding model in format "model_name@factory"
+        tenant_id (str): Tenant identifier for access control
+
+    Returns:
+        tuple[bool, Response | None]:
+        - First element (bool):
+            - True: Model is available and authorized
+            - False: Validation failed
+        - Second element contains:
+            - None on success
+            - Error detail dict on failure
+
+    Raises:
+        ValueError: When model identifier format is invalid
+        OperationalError: When database connection fails (auto-handled)
+
+    Examples:
+        >>> verify_embedding_availability("text-embedding@openai", "tenant_123")
+        (True, None)
+
+        >>> verify_embedding_availability("invalid_model", "tenant_123")
+        (False, {'code': 101, 'message': "Unsupported model: <invalid_model>"})
+    """
+    try:
+        llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(embd_id)
+        in_llm_service = bool(LLMService.query(llm_name=llm_name, fid=llm_factory, model_type="embedding"))
+
+        tenant_llms = TenantLLMService.get_my_llms(tenant_id=tenant_id)
+        is_tenant_model = any(llm["llm_name"] == llm_name and llm["llm_factory"] == llm_factory and llm["model_type"] == "embedding" for llm in tenant_llms)
+
+        is_builtin_model = embd_id in settings.BUILTIN_EMBEDDING_MODELS
+        if not (is_builtin_model or is_tenant_model or in_llm_service):
+            return False, get_error_argument_result(f"Unsupported model: <{embd_id}>")
+
+        if not (is_builtin_model or is_tenant_model):
+            return False, get_error_argument_result(f"Unauthorized model: <{embd_id}>")
+    except OperationalError as e:
+        logging.exception(e)
+        return False, get_error_data_result(message="Database operation failed")
+
+    return True, None
+
+
+def deep_merge(default: dict, custom: dict) -> dict:
+    """
+    Recursively merges two dictionaries with priority given to `custom` values.
+
+    Creates a deep copy of the `default` dictionary and iteratively merges nested
+    dictionaries using a stack-based approach. Non-dict values in `custom` will
+    completely override corresponding entries in `default`.
+
+    Args:
+        default (dict): Base dictionary containing default values.
+        custom (dict): Dictionary containing overriding values.
+
+    Returns:
+        dict: New merged dictionary combining values from both inputs.
+
+    Example:
+        >>> from copy import deepcopy
+        >>> default = {"a": 1, "nested": {"x": 10, "y": 20}}
+        >>> custom = {"b": 2, "nested": {"y": 99, "z": 30}}
+        >>> deep_merge(default, custom)
+        {'a': 1, 'b': 2, 'nested': {'x': 10, 'y': 99, 'z': 30}}
+
+        >>> deep_merge({"config": {"mode": "auto"}}, {"config": "manual"})
+        {'config': 'manual'}
+
+    Notes:
+        1. Merge priority is always given to `custom` values at all nesting levels
+        2. Non-dict values (e.g. list, str) in `custom` will replace entire values
+           in `default`, even if the original value was a dictionary
+        3. Time complexity: O(N) where N is total key-value pairs in `custom`
+        4. Recommended for configuration merging and nested data updates
+    """
+    merged = deepcopy(default)
+    stack = [(merged, custom)]
+
+    while stack:
+        base_dict, override_dict = stack.pop()
+
+        for key, val in override_dict.items():
+            if key in base_dict and isinstance(val, dict) and isinstance(base_dict[key], dict):
+                stack.append((base_dict[key], val))
+            else:
+                base_dict[key] = val
+
+    return merged
+
+
+def remap_dictionary_keys(source_data: dict, key_aliases: dict = None) -> dict:
+    """
+    Transform dictionary keys using a configurable mapping schema.
+
+    Args:
+        source_data: Original dictionary to process
+        key_aliases: Custom key transformation rules (Optional)
+            When provided, overrides default key mapping
+            Format: {<original_key>: <new_key>, ...}
+
+    Returns:
+        dict: New dictionary with transformed keys preserving original values
+
+    Example:
+        >>> input_data = {"old_key": "value", "another_field": 42}
+        >>> remap_dictionary_keys(input_data, {"old_key": "new_key"})
+        {'new_key': 'value', 'another_field': 42}
+    """
+    DEFAULT_KEY_MAP = {
+        "chunk_num": "chunk_count",
+        "doc_num": "document_count",
+        "parser_id": "chunk_method",
+        "embd_id": "embedding_model",
+    }
+
+    transformed_data = {}
+    mapping = key_aliases or DEFAULT_KEY_MAP
+
+    for original_key, value in source_data.items():
+        mapped_key = mapping.get(original_key, original_key)
+        transformed_data[mapped_key] = value
+
+    return transformed_data