Fix: catch non-begin component output (#7827)

### What problem does this PR solve?

Catch non-begin component output

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

.gitattributes

@@ -1 +1,2 @@
-*.sh text eol=lf
+*.sh text eol=lf
+docker/entrypoint.sh text eol=lf executable

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,15 +1,21 @@
-name: Bug Report
+name: "🐞 Bug Report"
 description: Create a bug issue for RAGFlow
 title: "[Bug]: "
-labels: [bug]
+labels: ["🐞 bug"]
 body:
 - type: checkboxes
   attributes:
-    label: Is there an existing issue for the same bug?
-    description: Please check if an issue already exists for the bug you encountered.
+    label: Self Checks
+    description: "Please check the following in order to be responded in time :)"
     options:
-    - label: I have checked the existing issues.
-      required: true
+      - label: I have searched for existing issues [search for existing issues](https://github.com/infiniflow/ragflow/issues), including closed ones.
+        required: true
+      - label: I confirm that I am using English to submit this report ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+        required: true
+      - label: Non-english title submitions will be closed directly ( 非英文标题的提交将会被直接关闭 ) ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+        required: true
+      - label: "Please do not modify this template :) and fill in all the required fields."
+        required: true
 - type: markdown
   attributes:
     value: "Please provide the following information to help us understand the issue."

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,10 +0,0 @@
----
-name: Feature request
-title: '[Feature Request]: '
-about: Suggest an idea for RAGFlow
-labels: ''
----
-
-**Summary**
-
-Description for this feature.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,14 +1,20 @@
-name: Feature request
+name: "💞 Feature request"
 description: Propose a feature request for RAGFlow.
 title: "[Feature Request]: "
-labels: [feature request]
+labels: ["💞 feature"]
 body:
   - type: checkboxes
     attributes:
-      label: Is there an existing issue for the same feature request?
-      description: Please check if an issue already exists for the feature you request.
+      label: Self Checks
+      description: "Please check the following in order to be responded in time :)"
       options:
-        - label: I have checked the existing issues.
+        - label: I have searched for existing issues [search for existing issues](https://github.com/infiniflow/ragflow/issues), including closed ones.
+          required: true
+        - label: I confirm that I am using English to submit this report ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+          required: true
+        - label: Non-english title submitions will be closed directly ( 非英文标题的提交将会被直接关闭 ) ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+          required: true
+        - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea
     attributes:

.github/ISSUE_TEMPLATE/question.yml

@@ -1,8 +1,21 @@
-name: Question
+name: "🙋‍♀️ Question"
 description: Ask questions on RAGFlow
 title: "[Question]: "
-labels: [question]
+labels: ["🙋‍♀️ question"]
 body:
+- type: checkboxes
+  attributes:
+    label: Self Checks
+    description: "Please check the following in order to be responded in time :)"
+    options:
+      - label: I have searched for existing issues [search for existing issues](https://github.com/infiniflow/ragflow/issues), including closed ones.
+        required: true
+      - label: I confirm that I am using English to submit this report ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+        required: true
+      - label: Non-english title submitions will be closed directly ( 非英文标题的提交将会被直接关闭 ) ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+        required: true
+      - label: "Please do not modify this template :) and fill in all the required fields."
+        required: true
 - type: markdown
   attributes:
     value: |

.github/workflows/release.yml

@@ -0,0 +1,118 @@
+name: release
+
+on:
+  schedule:
+    - cron: '0 13 * * *'  # This schedule runs every 13:00:00Z(21:00:00+08:00)
+  # The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags.
+  create:
+    tags:
+      - "v*.*.*"                  # normal release
+      - "nightly"                 # the only one mutable tag
+
+# https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  release:
+    runs-on: [ "self-hosted", "overseas" ]
+    steps:
+      - name: Ensure workspace ownership
+        run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
+
+      # https://github.com/actions/checkout/blob/v3/README.md
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Prepare release body
+        run: |
+          if [[ $GITHUB_EVENT_NAME == 'create' ]]; then
+            RELEASE_TAG=${GITHUB_REF#refs/tags/}
+            if [[ $RELEASE_TAG == 'nightly' ]]; then
+              PRERELEASE=true
+            else
+              PRERELEASE=false
+            fi
+            echo "Workflow triggered by create tag: $RELEASE_TAG"
+          else
+            RELEASE_TAG=nightly
+            PRERELEASE=true
+            echo "Workflow triggered by schedule"
+          fi
+          echo "RELEASE_TAG=$RELEASE_TAG" >> $GITHUB_ENV
+          echo "PRERELEASE=$PRERELEASE" >> $GITHUB_ENV
+          RELEASE_DATETIME=$(date --rfc-3339=seconds)
+          echo Release $RELEASE_TAG created from $GITHUB_SHA at $RELEASE_DATETIME > release_body.md
+
+      - name: Move the existing mutable tag
+        # https://github.com/softprops/action-gh-release/issues/171
+        run: |
+          git fetch --tags
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            # Determine if a given tag exists and matches a specific Git commit.
+            # actions/checkout@v4 fetch-tags doesn't work when triggered by schedule
+            if [ "$(git rev-parse -q --verify "refs/tags/$RELEASE_TAG")" = "$GITHUB_SHA" ]; then
+              echo "mutable tag $RELEASE_TAG exists and matches $GITHUB_SHA"
+            else
+              git tag -f $RELEASE_TAG $GITHUB_SHA
+              git push -f origin $RELEASE_TAG:refs/tags/$RELEASE_TAG
+              echo "created/moved mutable tag $RELEASE_TAG to $GITHUB_SHA"
+            fi
+          fi
+
+      - name: Create or overwrite a release
+        # https://github.com/actions/upload-release-asset has been replaced by https://github.com/softprops/action-gh-release
+        uses: softprops/action-gh-release@v2
+        with:
+          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+          prerelease: ${{ env.PRERELEASE }}
+          tag_name: ${{ env.RELEASE_TAG }}
+          # The body field does not support environment variable substitution directly.
+          body_path: release_body.md
+
+      # https://github.com/marketplace/actions/docker-login
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: infiniflow
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # https://github.com/marketplace/actions/build-and-push-docker-images
+      - name: Build and push full image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}
+          file: Dockerfile
+          platforms: linux/amd64
+
+      # https://github.com/marketplace/actions/build-and-push-docker-images
+      - name: Build and push slim image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}-slim
+          file: Dockerfile
+          build-args: LIGHTEN=1
+          platforms: linux/amd64
+
+      - name: Build ragflow-sdk
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          cd sdk/python && \
+          uv build
+
+      - name: Publish package distributions to PyPI
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: sdk/python/dist/
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          verbose: true

.github/workflows/tests.yml

@@ -15,6 +15,8 @@ on:
       - 'docs/**'
       - '*.md'
       - '*.mdx'
+  schedule:
+    - cron: '0 16 * * *'  # This schedule runs every 16:00:00Z(00:00:00+08:00)
 
 # https://docs.github.com/en/actions/using-jobs/using-concurrency
 concurrency:
@@ -32,12 +34,9 @@ jobs:
       # https://github.com/hmarr/debug-action
       #- uses: hmarr/debug-action@v2
 
-      - name: Show PR labels
+      - name: Show who triggered this workflow
         run: |
           echo "Workflow triggered by ${{ github.event_name }}"
-          if [[ ${{ github.event_name }} == 'pull_request' ]]; then
-            echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}"
-          fi
 
       - name: Ensure workspace ownership
         run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
@@ -49,29 +48,36 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
-      - name: Build ragflow:dev-slim
+      # https://github.com/astral-sh/ruff-action
+      - name: Static check with Ruff
+        uses: astral-sh/ruff-action@v3
+        with:
+          version: ">=0.11.x"
+          args: "check"
+
+      - name: Build ragflow:nightly-slim
         run: |
           RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
-          cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data ${RUNNER_WORKSPACE_PREFIX}/libssl*.deb  ${RUNNER_WORKSPACE_PREFIX}/tika-server*.jar* ${RUNNER_WORKSPACE_PREFIX}/chrome* ${RUNNER_WORKSPACE_PREFIX}/cl100k_base.tiktoken .
           sudo docker pull ubuntu:22.04
-          sudo docker build --progress=plain -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+          sudo docker build --progress=plain --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 
-      - name: Build ragflow:dev
+      - name: Build ragflow:nightly
         run: |
-          sudo docker build --progress=plain -f Dockerfile -t infiniflow/ragflow:dev .
+          sudo docker build --progress=plain --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
 
-      - name: Start ragflow:dev-slim
+      - name: Start ragflow:nightly-slim
         run: |
+          echo -e "\nRAGFLOW_IMAGE=infiniflow/ragflow:nightly-slim" >> docker/.env
           sudo docker compose -f docker/docker-compose.yml up -d
 
-      - name: Stop ragflow:dev-slim
+      - name: Stop ragflow:nightly-slim
         if: always()  # always run this step even if previous steps failed
         run: |
           sudo docker compose -f docker/docker-compose.yml down -v
 
-      - name: Start ragflow:dev
+      - name: Start ragflow:nightly
         run: |
-          echo "RAGFLOW_IMAGE=infiniflow/ragflow:dev" >> docker/.env
+          echo -e "\nRAGFLOW_IMAGE=infiniflow/ragflow:nightly" >> docker/.env
           sudo docker compose -f docker/docker-compose.yml up -d
 
       - name: Run sdk tests against Elasticsearch
@@ -82,7 +88,7 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && poetry install && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
+          cd sdk/python && uv sync --python 3.10 --group test --frozen && uv pip install . && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
 
       - name: Run frontend api tests against Elasticsearch
         run: |
@@ -92,15 +98,29 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && poetry install && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py 
+          cd sdk/python && uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
           
+      - name: Run http api tests against Elasticsearch
+        run: |
+          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
+          export HOST_ADDRESS=http://host.docker.internal:9380
+          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+            echo "Waiting for service to be available..."
+            sleep 5
+          done
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          cd sdk/python && uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_http_api && pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL}
 
-      - name: Stop ragflow:dev
+      - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
         run: |
           sudo docker compose -f docker/docker-compose.yml down -v
 
-      - name: Start ragflow:dev
+      - name: Start ragflow:nightly
         run: |
           sudo DOC_ENGINE=infinity docker compose -f docker/docker-compose.yml up -d
 
@@ -112,7 +132,7 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && poetry install && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
+          cd sdk/python && uv sync --python 3.10 --group test --frozen && uv pip install . && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
 
       - name: Run frontend api tests against Infinity
         run: |
@@ -122,9 +142,24 @@ jobs:
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && poetry install && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
+          cd sdk/python && uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
 
-      - name: Stop ragflow:dev
+      - name: Run http api tests against Infinity
+        run: |
+          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
+          export HOST_ADDRESS=http://host.docker.internal:9380
+          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+            echo "Waiting for service to be available..."
+            sleep 5
+          done
+          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          cd sdk/python && uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_http_api && DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL}
+
+      - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
         run: |
           sudo DOC_ENGINE=infinity docker compose -f docker/docker-compose.yml down -v

.gitignore

@@ -35,4 +35,155 @@ rag/res/deepdoc
 sdk/python/ragflow.egg-info/
 sdk/python/build/
 sdk/python/dist/
-sdk/python/ragflow_sdk.egg-info/
+sdk/python/ragflow_sdk.egg-info/
+huggingface.co/
+nltk_data/
+
+# Exclude hash-like temporary files like 9b5ad71b2ce5302211f9c61530b329a4922fc6a4
+*[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*
+.lh/
+.venv
+docker/data
+
+
+#--------------------------------------------------#
+# The following was generated with gitignore.nvim: #
+#--------------------------------------------------#
+# Gitignore for the following technologies: Node
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# Serverless Webpack directories
+.webpack/
+
+# SvelteKit build / generate output
+.svelte-kit
+

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-yaml
+      - id: check-json
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: mixed-line-ending
+      - id: check-symlinks
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.6
+    hooks:
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format

Dockerfile

@@ -3,55 +3,140 @@ FROM ubuntu:22.04 AS base
 USER root
 SHELL ["/bin/bash", "-c"]
 
-ENV LIGHTEN=0
+ARG NEED_MIRROR=0
+ARG LIGHTEN=0
+ENV LIGHTEN=${LIGHTEN}
 
 WORKDIR /ragflow
 
-RUN rm -f /etc/apt/apt.conf.d/docker-clean \
-    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-
-RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
-    apt update && apt-get --no-install-recommends install -y ca-certificates
-
-# Setup apt mirror site
-RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
-
-RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
-    apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus default-jdk python3-pip pipx \
-    libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils unzip libgbm-dev wget git \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && pip3 config set global.trusted-host "pypi.tuna.tsinghua.edu.cn mirrors.pku.edu.cn" && pip3 config set global.extra-index-url "https://mirrors.pku.edu.cn/pypi/web/simple" \
-    && pipx install poetry \
-    && /root/.local/bin/poetry self add poetry-plugin-pypi-mirror
-
-# https://forum.aspose.com/t/aspose-slides-for-net-no-usable-version-of-libssl-found-with-linux-server/271344/13
-# aspose-slides on linux/arm64 is unavailable
-RUN --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_amd64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-    --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_arm64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_arm64.deb \
-    if [ "$(uname -m)" = "x86_64" ]; then \
-        dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb; \
-    elif [ "$(uname -m)" = "aarch64" ]; then \
-        dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_arm64.deb; \
+# Copy models downloaded via download_deps.py
+RUN mkdir -p /ragflow/rag/res/deepdoc /root/.ragflow
+RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co,target=/huggingface.co \
+    cp /huggingface.co/InfiniFlow/huqie/huqie.txt.trie /ragflow/rag/res/ && \
+    tar --exclude='.*' -cf - \
+        /huggingface.co/InfiniFlow/text_concat_xgb_v1.0 \
+        /huggingface.co/InfiniFlow/deepdoc \
+        | tar -xf - --strip-components=3 -C /ragflow/rag/res/deepdoc 
+RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co,target=/huggingface.co \
+    if [ "$LIGHTEN" != "1" ]; then \
+        (tar -cf - \
+            /huggingface.co/BAAI/bge-large-zh-v1.5 \
+            /huggingface.co/maidalun1020/bce-embedding-base_v1 \
+            | tar -xf - --strip-components=2 -C /root/.ragflow) \
     fi
 
+# https://github.com/chrismattmann/tika-python
+# This is the only way to run python-tika without internet access. Without this set, the default is to check the tika version and pull latest every time from Apache.
+RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/deps \
+    cp -r /deps/nltk_data /root/ && \
+    cp /deps/tika-server-standard-3.0.0.jar /deps/tika-server-standard-3.0.0.jar.md5 /ragflow/ && \
+    cp /deps/cl100k_base.tiktoken /ragflow/9b5ad71b2ce5302211f9c61530b329a4922fc6a4
+
+ENV TIKA_SERVER_JAR="file:///ragflow/tika-server-standard-3.0.0.jar"
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Setup apt
+# Python package and implicit dependencies:
+# opencv-python: libglib2.0-0 libglx-mesa0 libgl1
+# aspose-slides: pkg-config libicu-dev libgdiplus         libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+# python-pptx:   default-jdk                              tika-server-standard-3.0.0.jar
+# selenium:      libatk-bridge2.0-0                       chrome-linux64-121-0-6167-85
+# Building C extensions: libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev
+RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
+    if [ "$NEED_MIRROR" == "1" ]; then \
+        sed -i 's|http://ports.ubuntu.com|http://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list; \
+        sed -i 's|http://archive.ubuntu.com|http://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list; \
+    fi; \
+    rm -f /etc/apt/apt.conf.d/docker-clean && \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache && \
+    chmod 1777 /tmp && \
+    apt update && \
+    apt --no-install-recommends install -y ca-certificates && \
+    apt update && \
+    apt install -y libglib2.0-0 libglx-mesa0 libgl1 && \
+    apt install -y pkg-config libicu-dev libgdiplus && \
+    apt install -y default-jdk && \
+    apt install -y libatk-bridge2.0-0 && \
+    apt install -y libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev && \
+    apt install -y libjemalloc-dev && \
+    apt install -y python3-pip pipx nginx unzip curl wget git vim less && \
+    apt install -y ghostscript
+
+RUN if [ "$NEED_MIRROR" == "1" ]; then \
+        pip3 config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
+        pip3 config set global.trusted-host mirrors.aliyun.com; \
+        mkdir -p /etc/uv && \
+        echo "[[index]]" > /etc/uv/uv.toml && \
+        echo 'url = "https://mirrors.aliyun.com/pypi/simple"' >> /etc/uv/uv.toml && \
+        echo "default = true" >> /etc/uv/uv.toml; \
+    fi; \
+    pipx install uv
+
 ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
 ENV PATH=/root/.local/bin:$PATH
-# Configure Poetry
-ENV POETRY_NO_INTERACTION=1
-ENV POETRY_VIRTUALENVS_IN_PROJECT=true
-ENV POETRY_VIRTUALENVS_CREATE=true
-ENV POETRY_REQUESTS_TIMEOUT=15
-ENV POETRY_PYPI_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple/
 
 # nodejs 12.22 on Ubuntu 22.04 is too old
-RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
-    apt purge -y nodejs npm && \
-    apt autoremove && \
+    apt purge -y nodejs npm cargo && \
+    apt autoremove -y && \
     apt update && \
-    apt install -y nodejs cargo && \
-    rm -rf /var/lib/apt/lists/*
+    apt install -y nodejs
+
+# A modern version of cargo is needed for the latest version of the Rust compiler.
+RUN apt update && apt install -y curl build-essential \
+    && if [ "$NEED_MIRROR" == "1" ]; then \
+         # Use TUNA mirrors for rustup/rust dist files
+         export RUSTUP_DIST_SERVER="https://mirrors.tuna.tsinghua.edu.cn/rustup"; \
+         export RUSTUP_UPDATE_ROOT="https://mirrors.tuna.tsinghua.edu.cn/rustup/rustup"; \
+         echo "Using TUNA mirrors for Rustup."; \
+       fi; \
+    # Force curl to use HTTP/1.1
+    curl --proto '=https' --tlsv1.2 --http1.1 -sSf https://sh.rustup.rs | bash -s -- -y --profile minimal \
+    && echo 'export PATH="/root/.cargo/bin:${PATH}"' >> /root/.bashrc
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN cargo --version && rustc --version
+
+# Add msssql ODBC driver
+# macOS ARM64 environment, install msodbcsql18.
+# general x86_64 environment, install msodbcsql17.
+RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
+    curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
+    curl https://packages.microsoft.com/config/ubuntu/22.04/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
+    apt update && \
+    arch="$(uname -m)"; \
+    if [ "$arch" = "arm64" ] || [ "$arch" = "aarch64" ]; then \
+        # ARM64 (macOS/Apple Silicon or Linux aarch64)
+        ACCEPT_EULA=Y apt install -y unixodbc-dev msodbcsql18; \
+    else \
+        # x86_64 or others
+        ACCEPT_EULA=Y apt install -y unixodbc-dev msodbcsql17; \
+    fi || \
+    { echo "Failed to install ODBC driver"; exit 1; }
+
+
+
+# Add dependencies of selenium
+RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/chrome-linux64-121-0-6167-85,target=/chrome-linux64.zip \
+    unzip /chrome-linux64.zip && \
+    mv chrome-linux64 /opt/chrome && \
+    ln -s /opt/chrome/chrome /usr/local/bin/
+RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/chromedriver-linux64-121-0-6167-85,target=/chromedriver-linux64.zip \
+    unzip -j /chromedriver-linux64.zip chromedriver-linux64/chromedriver && \
+    mv chromedriver /usr/local/bin/ && \
+    rm -f /usr/bin/google-chrome
+
+# https://forum.aspose.com/t/aspose-slides-for-net-no-usable-version-of-libssl-found-with-linux-server/271344/13
+# aspose-slides on linux/arm64 is unavailable
+RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/deps \
+    if [ "$(uname -m)" = "x86_64" ]; then \
+        dpkg -i /deps/libssl1.1_1.1.1f-1ubuntu2_amd64.deb; \
+    elif [ "$(uname -m)" = "aarch64" ]; then \
+        dpkg -i /deps/libssl1.1_1.1.1f-1ubuntu2_arm64.deb; \
+    fi
+
 
 # builder stage
 FROM base AS builder
@@ -59,17 +144,31 @@ USER root
 
 WORKDIR /ragflow
 
+# install dependencies from uv.lock file
+COPY pyproject.toml uv.lock ./
+
+# https://github.com/astral-sh/uv/issues/10462
+# uv records index url into uv.lock but doesn't failover among multiple indexes
+RUN --mount=type=cache,id=ragflow_uv,target=/root/.cache/uv,sharing=locked \
+    if [ "$NEED_MIRROR" == "1" ]; then \
+        sed -i 's|pypi.org|mirrors.aliyun.com/pypi|g' uv.lock; \
+    else \
+        sed -i 's|mirrors.aliyun.com/pypi|pypi.org|g' uv.lock; \
+    fi; \
+    if [ "$LIGHTEN" == "1" ]; then \
+        uv sync --python 3.10 --frozen; \
+    else \
+        uv sync --python 3.10 --frozen --all-extras; \
+    fi
+
+COPY web web
+COPY docs docs
+RUN --mount=type=cache,id=ragflow_npm,target=/root/.npm,sharing=locked \
+    cd web && npm install && npm run build
+
 COPY .git /ragflow/.git
 
-RUN current_commit=$(git rev-parse --short HEAD); \
-    last_tag=$(git describe --tags --abbrev=0); \
-    commit_count=$(git rev-list --count "$last_tag..HEAD"); \
-    version_info=""; \
-    if [ "$commit_count" -eq 0 ]; then \
-        version_info=$last_tag; \
-    else \
-        version_info="$current_commit($last_tag~$commit_count)"; \
-    fi; \
+RUN version_info=$(git describe --tags --match=v* --first-parent --always); \
     if [ "$LIGHTEN" == "1" ]; then \
         version_info="$version_info slim"; \
     else \
@@ -78,84 +177,12 @@ RUN current_commit=$(git rev-parse --short HEAD); \
     echo "RAGFlow version: $version_info"; \
     echo $version_info > /ragflow/VERSION
 
-COPY web web
-COPY docs docs
-RUN --mount=type=cache,id=ragflow_builder_npm,target=/root/.npm,sharing=locked \
-    cd web && npm install --force && npm run build
-
-# install dependencies from poetry.lock file
-COPY pyproject.toml poetry.toml poetry.lock ./
-
-RUN --mount=type=cache,id=ragflow_builder_poetry,target=/root/.cache/pypoetry,sharing=locked \
-    if [ "$LIGHTEN" == "1" ]; then \
-        poetry install --no-root; \
-    else \
-        poetry install --no-root --with=full; \
-    fi
-
 # production stage
 FROM base AS production
 USER root
 
 WORKDIR /ragflow
 
-COPY --from=builder /ragflow/VERSION /ragflow/VERSION
-
-# Install python packages' dependencies
-# cv2 requires libGL.so.1
-RUN --mount=type=cache,id=ragflow_production_apt,target=/var/cache/apt,sharing=locked \
-    apt update && apt install -y --no-install-recommends nginx libgl1 vim less && \
-    rm -rf /var/lib/apt/lists/*
-
-COPY web web
-COPY api api
-COPY conf conf
-COPY deepdoc deepdoc
-COPY rag rag
-COPY agent agent
-COPY graphrag graphrag
-COPY pyproject.toml poetry.toml poetry.lock ./
-
-# Copy models downloaded via download_deps.py
-RUN mkdir -p /ragflow/rag/res/deepdoc /root/.ragflow
-RUN --mount=type=bind,source=huggingface.co,target=/huggingface.co \
-    tar --exclude='.*' -cf - \
-        /huggingface.co/InfiniFlow/text_concat_xgb_v1.0 \
-        /huggingface.co/InfiniFlow/deepdoc \
-        | tar -xf - --strip-components=3 -C /ragflow/rag/res/deepdoc
-RUN --mount=type=bind,source=huggingface.co,target=/huggingface.co \
-    tar -cf - \
-        /huggingface.co/BAAI/bge-large-zh-v1.5 \
-        /huggingface.co/BAAI/bge-reranker-v2-m3 \
-        /huggingface.co/maidalun1020/bce-embedding-base_v1 \
-        /huggingface.co/maidalun1020/bce-reranker-base_v1 \
-        | tar -xf - --strip-components=2 -C /root/.ragflow
-
-# Copy nltk data downloaded via download_deps.py
-COPY nltk_data /root/nltk_data
-
-# https://github.com/chrismattmann/tika-python
-# This is the only way to run python-tika without internet access. Without this set, the default is to check the tika version and pull latest every time from Apache.
-COPY tika-server-standard-3.0.0.jar /ragflow/tika-server-standard.jar
-COPY tika-server-standard-3.0.0.jar.md5 /ragflow/tika-server-standard.jar.md5
-ENV TIKA_SERVER_JAR="file:///ragflow/tika-server-standard.jar"
-
-# Copy cl100k_base
-COPY cl100k_base.tiktoken /ragflow/9b5ad71b2ce5302211f9c61530b329a4922fc6a4
-
-# Add dependencies of selenium
-RUN --mount=type=bind,source=chrome-linux64-121-0-6167-85,target=/chrome-linux64.zip \
-    unzip /chrome-linux64.zip && \
-    mv chrome-linux64 /opt/chrome && \
-    ln -s /opt/chrome/chrome /usr/local/bin/
-RUN --mount=type=bind,source=chromedriver-linux64-121-0-6167-85,target=/chromedriver-linux64.zip \
-    unzip -j /chromedriver-linux64.zip chromedriver-linux64/chromedriver && \
-    mv chromedriver /usr/local/bin/ && \
-    rm -f /usr/bin/google-chrome
-
-# Copy compiled web pages
-COPY --from=builder /ragflow/web/dist /ragflow/web/dist
-
 # Copy Python environment and packages
 ENV VIRTUAL_ENV=/ragflow/.venv
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
@@ -163,8 +190,24 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 
 ENV PYTHONPATH=/ragflow/
 
-COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template
-COPY docker/entrypoint.sh ./entrypoint.sh
-RUN chmod +x ./entrypoint.sh
+COPY web web
+COPY api api
+COPY conf conf
+COPY deepdoc deepdoc
+COPY rag rag
+COPY agent agent
+COPY graphrag graphrag
+COPY agentic_reasoning agentic_reasoning
+COPY pyproject.toml uv.lock ./
+COPY mcp mcp
+COPY plugin plugin
 
+COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template
+COPY docker/entrypoint.sh ./
+RUN chmod +x ./entrypoint*.sh
+
+# Copy compiled web pages
+COPY --from=builder /ragflow/web/dist /ragflow/web/dist
+
+COPY --from=builder /ragflow/VERSION /ragflow/VERSION
 ENTRYPOINT ["./entrypoint.sh"]

Dockerfile.deps

@@ -0,0 +1,10 @@
+# This builds an image that contains the resources needed by Dockerfile
+#
+FROM scratch
+
+# Copy resources downloaded via download_deps.py
+COPY chromedriver-linux64-121-0-6167-85 chrome-linux64-121-0-6167-85 cl100k_base.tiktoken libssl1.1_1.1.1f-1ubuntu2_amd64.deb libssl1.1_1.1.1f-1ubuntu2_arm64.deb tika-server-standard-3.0.0.jar tika-server-standard-3.0.0.jar.md5 libssl*.deb /
+
+COPY nltk_data /nltk_data
+
+COPY huggingface.co /huggingface.co

Dockerfile.scratch.oc9

@@ -33,6 +33,7 @@ ADD ./rag ./rag
 ADD ./requirements.txt ./requirements.txt
 ADD ./agent ./agent
 ADD ./graphrag ./graphrag
+ADD ./plugin ./plugin
 
 RUN dnf install -y openmpi openmpi-devel python3-openmpi
 ENV C_INCLUDE_PATH /usr/include/openmpi-x86_64:$C_INCLUDE_PATH

Dockerfile.slim

@@ -1,163 +0,0 @@
-# base stage
-FROM ubuntu:22.04 AS base
-USER root
-SHELL ["/bin/bash", "-c"]
-
-ENV LIGHTEN=1
-
-WORKDIR /ragflow
-
-RUN rm -f /etc/apt/apt.conf.d/docker-clean \
-    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-
-RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
-    apt update && apt-get --no-install-recommends install -y ca-certificates
-
-# Setup apt mirror site
-RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
-
-RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
-    apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus default-jdk python3-pip pipx \
-    libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils unzip libgbm-dev wget git \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && pip3 config set global.trusted-host "pypi.tuna.tsinghua.edu.cn mirrors.pku.edu.cn" && pip3 config set global.extra-index-url "https://mirrors.pku.edu.cn/pypi/web/simple" \
-    && pipx install poetry \
-    && /root/.local/bin/poetry self add poetry-plugin-pypi-mirror
-
-# https://forum.aspose.com/t/aspose-slides-for-net-no-usable-version-of-libssl-found-with-linux-server/271344/13
-# aspose-slides on linux/arm64 is unavailable
-RUN --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_amd64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-    --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_arm64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_arm64.deb \
-    if [ "$(uname -m)" = "x86_64" ]; then \
-        dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb; \
-    elif [ "$(uname -m)" = "aarch64" ]; then \
-        dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_arm64.deb; \
-    fi
-
-ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
-ENV PATH=/root/.local/bin:$PATH
-# Configure Poetry
-ENV POETRY_NO_INTERACTION=1
-ENV POETRY_VIRTUALENVS_IN_PROJECT=true
-ENV POETRY_VIRTUALENVS_CREATE=true
-ENV POETRY_REQUESTS_TIMEOUT=15
-ENV POETRY_PYPI_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple/
-
-# nodejs 12.22 on Ubuntu 22.04 is too old
-RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
-    curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
-    apt purge -y nodejs npm && \
-    apt autoremove && \
-    apt update && \
-    apt install -y nodejs cargo && \
-    rm -rf /var/lib/apt/lists/*
-
-# builder stage
-FROM base AS builder
-USER root
-
-WORKDIR /ragflow
-
-COPY .git /ragflow/.git
-
-RUN current_commit=$(git rev-parse --short HEAD); \
-    last_tag=$(git describe --tags --abbrev=0); \
-    commit_count=$(git rev-list --count "$last_tag..HEAD"); \
-    version_info=""; \
-    if [ "$commit_count" -eq 0 ]; then \
-        version_info=$last_tag; \
-    else \
-        version_info="$current_commit($last_tag~$commit_count)"; \
-    fi; \
-    if [ "$LIGHTEN" == "1" ]; then \
-        version_info="$version_info slim"; \
-    else \
-        version_info="$version_info full"; \
-    fi; \
-    echo "RAGFlow version: $version_info"; \
-    echo $version_info > /ragflow/VERSION
-
-COPY web web
-COPY docs docs
-RUN --mount=type=cache,id=ragflow_builder_npm,target=/root/.npm,sharing=locked \
-    cd web && npm install --force && npm run build
-
-# install dependencies from poetry.lock file
-COPY pyproject.toml poetry.toml poetry.lock ./
-
-RUN --mount=type=cache,id=ragflow_builder_poetry,target=/root/.cache/pypoetry,sharing=locked \
-    if [ "$LIGHTEN" == "1" ]; then \
-        poetry install --no-root; \
-    else \
-        poetry install --no-root --with=full; \
-    fi
-
-# production stage
-FROM base AS production
-USER root
-
-WORKDIR /ragflow
-
-COPY --from=builder /ragflow/VERSION /ragflow/VERSION
-
-# Install python packages' dependencies
-# cv2 requires libGL.so.1
-RUN --mount=type=cache,id=ragflow_production_apt,target=/var/cache/apt,sharing=locked \
-    apt update && apt install -y --no-install-recommends nginx libgl1 vim less && \
-    rm -rf /var/lib/apt/lists/*
-
-COPY web web
-COPY api api
-COPY conf conf
-COPY deepdoc deepdoc
-COPY rag rag
-COPY agent agent
-COPY graphrag graphrag
-COPY pyproject.toml poetry.toml poetry.lock ./
-
-# Copy models downloaded via download_deps.py
-RUN mkdir -p /ragflow/rag/res/deepdoc /root/.ragflow
-RUN --mount=type=bind,source=huggingface.co,target=/huggingface.co \
-    tar --exclude='.*' -cf - \
-        /huggingface.co/InfiniFlow/text_concat_xgb_v1.0 \
-        /huggingface.co/InfiniFlow/deepdoc \
-        | tar -xf - --strip-components=3 -C /ragflow/rag/res/deepdoc
-
-# Copy nltk data downloaded via download_deps.py
-COPY nltk_data /root/nltk_data
-
-# https://github.com/chrismattmann/tika-python
-# This is the only way to run python-tika without internet access. Without this set, the default is to check the tika version and pull latest every time from Apache.
-COPY tika-server-standard-3.0.0.jar /ragflow/tika-server-standard.jar
-COPY tika-server-standard-3.0.0.jar.md5 /ragflow/tika-server-standard.jar.md5
-ENV TIKA_SERVER_JAR="file:///ragflow/tika-server-standard.jar"
-
-# Copy cl100k_base
-COPY cl100k_base.tiktoken /ragflow/9b5ad71b2ce5302211f9c61530b329a4922fc6a4
-
-# Add dependencies of selenium
-RUN --mount=type=bind,source=chrome-linux64-121-0-6167-85,target=/chrome-linux64.zip \
-    unzip /chrome-linux64.zip && \
-    mv chrome-linux64 /opt/chrome && \
-    ln -s /opt/chrome/chrome /usr/local/bin/
-RUN --mount=type=bind,source=chromedriver-linux64-121-0-6167-85,target=/chromedriver-linux64.zip \
-    unzip -j /chromedriver-linux64.zip chromedriver-linux64/chromedriver && \
-    mv chromedriver /usr/local/bin/ && \
-    rm -f /usr/bin/google-chrome
-
-# Copy compiled web pages
-COPY --from=builder /ragflow/web/dist /ragflow/web/dist
-
-# Copy Python environment and packages
-ENV VIRTUAL_ENV=/ragflow/.venv
-COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
-
-ENV PYTHONPATH=/ragflow/
-
-COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template
-COPY docker/entrypoint.sh ./entrypoint.sh
-RUN chmod +x ./entrypoint.sh
-
-ENTRYPOINT ["./entrypoint.sh"]

README.md

@@ -7,9 +7,11 @@
 <p align="center">
   <a href="./README.md">English</a> |
   <a href="./README_zh.md">简体中文</a> |
+  <a href="./README_tzh.md">繁体中文</a> |
   <a href="./README_ja.md">日本語</a> |
   <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a>
+  <a href="./README_id.md">Bahasa Indonesia</a> |
+  <a href="/README_pt_br.md">Português (Brasil)</a>
 </p>
 
 <p align="center">
@@ -20,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.14.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.14.0">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -32,14 +34,14 @@
 
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
-  <a href="https://github.com/infiniflow/ragflow/issues/162">Roadmap</a> |
+  <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
 <details open>
-<summary></b>📕 Table of Contents</b></summary>
+<summary><b>📕 Table of Contents</b></summary>
 
 - 💡 [What is RAGFlow?](#-what-is-ragflow)
 - 🎮 [Demo](#-demo)
@@ -68,6 +70,7 @@ data.
 ## 🎮 Demo
 
 Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/infiniflow/ragflow/assets/7248/2f6baa3e-1092-4f11-866d-36f6a9d075e5" width="1200"/>
 <img src="https://github.com/user-attachments/assets/504bbbf1-c9f7-4d83-8cc5-e9cb63c26db6" width="1200"/>
@@ -75,16 +78,18 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Latest Updates
 
-- 2024-11-22 Adds more variables to Agent.
+- 2025-03-19 Supports using a multi-modal model to make sense of images within PDF or DOCX files.
+- 2025-02-28 Combined with Internet search (Tavily), supports reasoning like Deep Research for any LLMs.
+- 2025-01-26 Optimizes knowledge graph extraction and application, offering various configuration options.
+- 2024-12-18 Upgrades Document Layout Analysis model in DeepDoc.
 - 2024-11-01 Adds keyword extraction and related question generation to the parsed chunks to improve the accuracy of retrieval.
-- 2024-09-13 Adds search mode for knowledge base Q&A.
 - 2024-08-22 Support text to SQL statements through RAG.
-- 2024-08-02 Supports GraphRAG inspired by [graphrag](https://github.com/microsoft/graphrag) and mind map.
 
 ## 🎉 Stay Tuned
 
 ⭐️ Star our repository to stay up-to-date with exciting new features and improvements! Get instant notifications for new
 releases! 🌟
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
 </div>
@@ -132,8 +137,10 @@ releases! 🌟
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > If you have not installed Docker on your local machine (Windows, Mac, or Linux),
-  see [Install Docker Engine](https://docs.docker.com/engine/install/).
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): Required only if you intend to use the code executor (sandbox) feature of RAGFlow.
+
+> [!TIP]
+> If you have not installed Docker on your local machine (Windows, Mac, or Linux), see [Install Docker Engine](https://docs.docker.com/engine/install/).
 
 ### 🚀 Start up the server
 
@@ -153,7 +160,7 @@ releases! 🌟
    > ```
    >
    > This change will be reset after a system reboot. To ensure your change remains permanent, add or update the
-   `vm.max_map_count` value in **/etc/sysctl.conf** accordingly:
+   > `vm.max_map_count` value in **/etc/sysctl.conf** accordingly:
    >
    > ```bash
    > vm.max_map_count=262144
@@ -165,29 +172,29 @@ releases! 🌟
    $ git clone https://github.com/infiniflow/ragflow.git
    ```
 
-3. Build the pre-built Docker images and start up the server:
+3. Start up the server using the pre-built Docker images:
 
-   > The command below downloads the dev version Docker image for RAGFlow slim (`dev-slim`). Note that RAGFlow slim
-   Docker images do not include embedding models or Python libraries and hence are approximately 1GB in size.
+> [!CAUTION]
+> All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
+> If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
+
+   > The command below downloads the `v0.19.0-slim` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.19.0-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0` for the full edition `v0.19.0`.
 
    ```bash
    $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
    ```
 
-   > - To download a RAGFlow slim Docker image of a specific version, update the `RAGFLOW_IMAGE` variable in *
-       *docker/.env** to your desired version. For example, `RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0-slim`. After
-       making this change, rerun the command above to initiate the download.
-   > - To download the dev version of RAGFlow Docker image *including* embedding models and Python libraries, update the
-       `RAGFLOW_IMAGE` variable in **docker/.env** to `RAGFLOW_IMAGE=infiniflow/ragflow:dev`. After making this change,
-       rerun the command above to initiate the download.
-   > - To download a specific version of RAGFlow Docker image *including* embedding models and Python libraries, update
-       the `RAGFLOW_IMAGE` variable in **docker/.env** to your desired version. For example,
-       `RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0`. After making this change, rerun the command above to initiate the
-       download.
-
-   > **NOTE:** A RAGFlow Docker image that includes embedding models and Python libraries is approximately 9GB in size
-   and may take significantly longer time to load.
+   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+   |-------------------|-----------------|-----------------------|--------------------------|
+   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.0-slim      | &approx;2       | ❌                   | Stable release            |
+   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                   | _Unstable_ nightly build  |
 
 4. Check the server status after having the server up and running:
 
@@ -199,23 +206,21 @@ releases! 🌟
 
    ```bash
 
-         ____   ___    ______ ______ __               
+         ____   ___    ______ ______ __
         / __ \ /   |  / ____// ____// /____  _      __
        / /_/ // /| | / / __ / /_   / // __ \| | /| / /
-      / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ / 
-     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/ 
+      / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
+     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
+
    > If you skip this confirmation step and directly log in to RAGFlow, your browser may prompt a `network anormal`
-   error because, at that moment, your RAGFlow may not be fully initialized.
+   > error because, at that moment, your RAGFlow may not be fully initialized.
 
 5. In your web browser, enter the IP address of your server and log in to RAGFlow.
    > With the default settings, you only need to enter `http://IP_OF_YOUR_MACHINE` (**sans** port number) as the default
-   HTTP serving port `80` can be omitted when using the default configurations.
+   > HTTP serving port `80` can be omitted when using the default configurations.
 6. In [service_conf.yaml.template](./docker/service_conf.yaml.template), select the desired LLM factory in `user_default_llm` and update
    the `API_KEY` field with the corresponding API key.
 
@@ -241,7 +246,7 @@ to `<YOUR_SERVING_PORT>:80`.
 Updates to the above configurations require a reboot of all containers to take effect:
 
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ### Switch doc engine from Elasticsearch to Infinity
@@ -254,27 +259,28 @@ RAGFlow uses Elasticsearch by default for storing full text and vectors. To swit
    $ docker compose -f docker/docker-compose.yml down -v
    ```
 
+> [!WARNING]
+> `-v` will delete the docker container volumes, and the existing data will be cleared.
+
 2. Set `DOC_ENGINE` in **docker/.env** to `infinity`.
 
 3. Start the containers:
 
    ```bash
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
 
-> [!WARNING] 
+> [!WARNING]
 > Switching to Infinity on a Linux/arm64 machine is not yet officially supported.
 
 ## 🔧 Build a Docker image without embedding models
 
-This image is approximately 1 GB in size and relies on external LLM and embedding services.
+This image is approximately 2 GB in size and relies on external LLM and embedding services.
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 Build a Docker image including embedding models
@@ -284,36 +290,38 @@ This image is approximately 9 GB in size. As it includes embedding models, it re
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile -t infiniflow/ragflow:dev .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 Launch service from source for development
 
-1. Install Poetry, or skip this step if it is already installed:
+1. Install uv, or skip this step if it is already installed:
+
    ```bash
-   curl -sSL https://install.python-poetry.org | python3 -
+   pipx install uv pre-commit
    ```
 
 2. Clone the source code and install Python dependencies:
+
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true
-   ~/.local/bin/poetry install --sync --no-root --with=full # install RAGFlow dependent python modules
+   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. Launch the dependent services (MinIO, Elasticsearch, Redis, and MySQL) using Docker Compose:
+
    ```bash
    docker compose -f docker/docker-compose-base.yml up -d
    ```
 
    Add the following line to `/etc/hosts` to resolve all hosts specified in **docker/.env** to `127.0.0.1`:
+
+   ```
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-   127.0.0.1       es01 infinity mysql minio redis
-   ```  
-   In **docker/service_conf.yaml.template**, update mysql port to `5455` and es port to `1200`, as specified in **docker/.env**.
 
 4. If you cannot access HuggingFace, set the `HF_ENDPOINT` environment variable to use a mirror site:
 
@@ -321,46 +329,68 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev .
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. Launch backend service:
+5. If your operating system does not have jemalloc, please install it as follows:
+
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+   
+6. Launch backend service:
+
    ```bash
    source .venv/bin/activate
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
 
-6. Install frontend dependencies:
+7. Install frontend dependencies:
+
    ```bash
    cd web
-   npm install --force
-   ```  
-7. Configure frontend to update `proxy.target` in **.umirc.ts** to `http://127.0.0.1:9380`:
+   npm install
+   ```
+
 8. Launch frontend service:
+
    ```bash
-   npm run dev 
-   ```  
+   npm run dev
+   ```
 
    _The following output confirms a successful launch of the system:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+9. Stop RAGFlow front-end and back-end service after development is complete:
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
 ## 📚 Documentation
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/guides)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
 - [References](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 Roadmap
 
-See the [RAGFlow Roadmap 2024](https://github.com/infiniflow/ragflow/issues/162)
+See the [RAGFlow Roadmap 2025](https://github.com/infiniflow/ragflow/issues/4214)
 
 ## 🏄 Community
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/NjYzJD3GM3)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Contributing
 
 RAGFlow flourishes via open-source collaboration. In this spirit, we embrace diverse contributions from the community.
-If you would like to be a part, review our [Contribution Guidelines](./CONTRIBUTING.md) first.
+If you would like to be a part, review our [Contribution Guidelines](https://ragflow.io/docs/dev/contributing) first.

README_id.md

@@ -7,9 +7,11 @@
 <p align="center">
   <a href="./README.md">English</a> |
   <a href="./README_zh.md">简体中文</a> |
+  <a href="./README_tzh.md">繁体中文</a> |
   <a href="./README_ja.md">日本語</a> |
   <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a>
+  <a href="./README_id.md">Bahasa Indonesia</a> |
+  <a href="/README_pt_br.md">Português (Brasil)</a>
 </p>
 
 <p align="center">
@@ -20,7 +22,7 @@
         <img alt="Lencana Daring" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.14.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.14.0">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Rilis%20Terbaru" alt="Rilis Terbaru">
@@ -32,14 +34,14 @@
 
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Dokumentasi</a> |
-  <a href="https://github.com/infiniflow/ragflow/issues/162">Peta Jalan</a> |
+  <a href="https://github.com/infiniflow/ragflow/issues/4214">Peta Jalan</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
 <details open>
-<summary></b>📕 Daftar Isi</b></summary>
+<summary><b>📕 Daftar Isi </b> </summary>
 
 - 💡 [Apa Itu RAGFlow?](#-apa-itu-ragflow)
 - 🎮 [Demo](#-demo)
@@ -65,6 +67,7 @@
 ## 🎮 Demo
 
 Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/infiniflow/ragflow/assets/7248/2f6baa3e-1092-4f11-866d-36f6a9d075e5" width="1200"/>
 <img src="https://github.com/user-attachments/assets/504bbbf1-c9f7-4d83-8cc5-e9cb63c26db6" width="1200"/>
@@ -72,15 +75,17 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Pembaruan Terbaru
 
-- 22-11-2024 Peningkatan definisi dan penggunaan variabel di Agen.
-- 2024-11-01: Penambahan ekstraksi kata kunci dan pembuatan pertanyaan terkait untuk meningkatkan akurasi pengambilan.
-- 2024-09-13: Penambahan mode pencarian untuk Q&A basis pengetahuan.
-- 2024-08-22: Dukungan untuk teks ke pernyataan SQL melalui RAG.
-- 2024-08-02: Dukungan GraphRAG yang terinspirasi oleh [graphrag](https://github.com/microsoft/graphrag) dan mind map.
+- 2025-03-19 Mendukung penggunaan model multi-modal untuk memahami gambar di dalam file PDF atau DOCX.
+- 2025-02-28 dikombinasikan dengan pencarian Internet (TAVILY), mendukung penelitian mendalam untuk LLM apa pun.
+- 2025-01-26 Optimalkan ekstraksi dan penerapan grafik pengetahuan dan sediakan berbagai opsi konfigurasi.
+- 2024-12-18 Meningkatkan model Analisis Tata Letak Dokumen di DeepDoc.
+- 2024-11-01 Penambahan ekstraksi kata kunci dan pembuatan pertanyaan terkait untuk meningkatkan akurasi pengambilan.
+- 2024-08-22 Dukungan untuk teks ke pernyataan SQL melalui RAG.
 
 ## 🎉 Tetap Terkini
 
 ⭐️ Star repositori kami untuk tetap mendapat informasi tentang fitur baru dan peningkatan menarik! 🌟
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
 </div>
@@ -127,6 +132,10 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): Hanya diperlukan jika Anda ingin menggunakan fitur eksekutor kode (sandbox) dari RAGFlow.
+
+> [!TIP]
+> Jika Anda belum menginstal Docker di komputer lokal Anda (Windows, Mac, atau Linux), lihat [Install Docker Engine](https://docs.docker.com/engine/install/).
 
 ### 🚀 Menjalankan Server
 
@@ -146,7 +155,7 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
    > ```
    >
    > Perubahan ini akan hilang setelah sistem direboot. Untuk membuat perubahan ini permanen, tambahkan atau perbarui nilai
-   `vm.max_map_count` di **/etc/sysctl.conf**:
+   > `vm.max_map_count` di **/etc/sysctl.conf**:
    >
    > ```bash
    > vm.max_map_count=262144
@@ -160,28 +169,29 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
 3. Bangun image Docker pre-built dan jalankan server:
 
-   > Perintah di bawah ini akan mengunduh versi dev dari Docker image RAGFlow slim (`dev-slim`). Image RAGFlow slim
-   tidak termasuk model embedding atau library Python dan berukuran sekitar 1GB.
+> [!CAUTION]
+> Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
+> Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
 
-   ```bash
-   $ cd ragflow/docker
-   $ docker compose -f docker-compose.yml up -d
-   ```
+> Perintah di bawah ini mengunduh edisi v0.19.0-slim dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.19.0-slim, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server. Misalnya, atur RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0 untuk edisi lengkap v0.19.0.
 
-   > - Untuk mengunduh versi tertentu dari image Docker RAGFlow slim, perbarui variabel `RAGFlow_IMAGE` di *
-       *docker/.env** sesuai dengan versi yang diinginkan. Misalnya, `RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0-slim`.
-       Setelah mengubah ini, jalankan ulang perintah di atas untuk memulai unduhan.
-   > - Untuk mengunduh versi dev dari image Docker RAGFlow *termasuk* model embedding dan library Python, perbarui
-       variabel `RAGFlow_IMAGE` di **docker/.env** menjadi `RAGFLOW_IMAGE=infiniflow/ragflow:dev`. Setelah mengubah ini,
-       jalankan ulang perintah di atas untuk memulai unduhan.
-   > - Untuk mengunduh versi tertentu dari image Docker RAGFlow *termasuk* model embedding dan library Python, perbarui
-       variabel `RAGFlow_IMAGE` di **docker/.env** sesuai dengan versi yang diinginkan. Misalnya,
-       `RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0`. Setelah mengubah ini, jalankan ulang perintah di atas untuk memulai unduhan.
+```bash
+$ cd ragflow/docker
+# Use CPU for embedding and DeepDoc tasks:
+$ docker compose -f docker-compose.yml up -d
 
-   > **CATATAN:** Image Docker RAGFlow yang mencakup model embedding dan library Python berukuran sekitar 9GB
-   dan mungkin memerlukan waktu lebih lama untuk dimuat.
+# To use GPU to accelerate embedding and DeepDoc tasks:
+# docker compose -f docker-compose-gpu.yml up -d
+```
 
-4. Periksa status server setelah server aktif dan berjalan:
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+| ----------------- | --------------- | --------------------- | ------------------------ |
+| v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
+| v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+| nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
+| nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
+
+1. Periksa status server setelah server aktif dan berjalan:
 
    ```bash
    $ docker logs -f ragflow-server
@@ -191,24 +201,22 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
    ```bash
 
-         ____   ___    ______ ______ __               
+         ____   ___    ______ ______ __
         / __ \ /   |  / ____// ____// /____  _      __
        / /_/ // /| | / / __ / /_   / // __ \| | /| / /
-      / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ / 
-     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/ 
+      / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
+     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
-   > Jika Anda melewatkan langkah ini dan langsung login ke RAGFlow, browser Anda mungkin menampilkan error `network anormal`
-   karena RAGFlow mungkin belum sepenuhnya siap.
 
-5. Buka browser web Anda, masukkan alamat IP server Anda, dan login ke RAGFlow.
-   > Dengan pengaturan default, Anda hanya perlu memasukkan `http://IP_DEVICE_ANDA` (**tanpa** nomor port) karena 
-   port HTTP default `80` bisa dihilangkan saat menggunakan konfigurasi default.
-6. Dalam [service_conf.yaml](./docker/service_conf.yaml), pilih LLM factory yang diinginkan di `user_default_llm` dan perbarui
+   > Jika Anda melewatkan langkah ini dan langsung login ke RAGFlow, browser Anda mungkin menampilkan error `network anormal`
+   > karena RAGFlow mungkin belum sepenuhnya siap.
+
+2. Buka browser web Anda, masukkan alamat IP server Anda, dan login ke RAGFlow.
+   > Dengan pengaturan default, Anda hanya perlu memasukkan `http://IP_DEVICE_ANDA` (**tanpa** nomor port) karena
+   > port HTTP default `80` bisa dihilangkan saat menggunakan konfigurasi default.
+3. Dalam [service_conf.yaml.template](./docker/service_conf.yaml.template), pilih LLM factory yang diinginkan di `user_default_llm` dan perbarui
    bidang `API_KEY` dengan kunci API yang sesuai.
 
    > Lihat [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) untuk informasi lebih lanjut.
@@ -221,35 +229,26 @@ Untuk konfigurasi sistem, Anda perlu mengelola file-file berikut:
 
 - [.env](./docker/.env): Menyimpan pengaturan dasar sistem, seperti `SVR_HTTP_PORT`, `MYSQL_PASSWORD`, dan
   `MINIO_PASSWORD`.
-- [service_conf.yaml](./docker/service_conf.yaml): Mengonfigurasi aplikasi backend.
+- [service_conf.yaml.template](./docker/service_conf.yaml.template): Mengonfigurasi aplikasi backend.
 - [docker-compose.yml](./docker/docker-compose.yml): Sistem ini bergantung pada [docker-compose.yml](./docker/docker-compose.yml) untuk memulai.
 
-Anda harus memastikan bahwa perubahan pada file [.env](./docker/.env) sesuai dengan yang ada di file [service_conf.yaml](./docker/service_conf.yaml).
-
-> File [./docker/README](./docker/README.md) menyediakan penjelasan detail tentang pengaturan lingkungan dan konfigurasi aplikasi,
-> dan Anda DIWAJIBKAN memastikan bahwa semua pengaturan lingkungan yang tercantum di
-> [./docker/README](./docker/README.md) selaras dengan konfigurasi yang sesuai di
-> [service_conf.yaml](./docker/service_conf.yaml).
-
 Untuk memperbarui port HTTP default (80), buka [docker-compose.yml](./docker/docker-compose.yml) dan ubah `80:80`
 menjadi `<YOUR_SERVING_PORT>:80`.
 
 Pembaruan konfigurasi ini memerlukan reboot semua kontainer agar efektif:
 
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ## 🔧 Membangun Docker Image tanpa Model Embedding
 
-Image ini berukuran sekitar 1 GB dan bergantung pada aplikasi LLM eksternal dan embedding.
+Image ini berukuran sekitar 2 GB dan bergantung pada aplikasi LLM eksternal dan embedding.
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 Membangun Docker Image Termasuk Model Embedding
@@ -259,36 +258,38 @@ Image ini berukuran sekitar 9 GB. Karena sudah termasuk model embedding, ia hany
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile -t infiniflow/ragflow:dev .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 Menjalankan Aplikasi dari untuk Pengembangan
 
-1. Instal Poetry, atau lewati langkah ini jika sudah terinstal:
+1. Instal uv, atau lewati langkah ini jika sudah terinstal:
+
    ```bash
-   curl -sSL https://install.python-poetry.org | python3 -
+   pipx install uv pre-commit
    ```
 
 2. Clone kode sumber dan instal dependensi Python:
+
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true
-   ~/.local/bin/poetry install --sync --no-root # install modul python RAGFlow
+   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. Jalankan aplikasi yang diperlukan (MinIO, Elasticsearch, Redis, dan MySQL) menggunakan Docker Compose:
+
    ```bash
    docker compose -f docker/docker-compose-base.yml up -d
    ```
 
-   Tambahkan baris berikut ke `/etc/hosts` untuk memetakan semua host yang ditentukan di **docker/service_conf.yaml** ke `127.0.0.1`:
+   Tambahkan baris berikut ke `/etc/hosts` untuk memetakan semua host yang ditentukan di **conf/service_conf.yaml** ke `127.0.0.1`:
+
+   ```
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-   127.0.0.1       es01 infinity mysql minio redis
-   ```  
-   Di **docker/service_conf.yaml**, perbarui port mysql ke `5455` dan es ke `1200`, sesuai dengan yang ditentukan di **docker/.env**.
 
 4. Jika Anda tidak dapat mengakses HuggingFace, atur variabel lingkungan `HF_ENDPOINT` untuk menggunakan situs mirror:
 
@@ -296,46 +297,69 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev .
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. Jalankan aplikasi backend:
+5. Jika sistem operasi Anda tidak memiliki jemalloc, instal sebagai berikut:
+
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+
+6. Jalankan aplikasi backend:
+
    ```bash
    source .venv/bin/activate
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
 
-6. Instal dependensi frontend:
+7. Instal dependensi frontend:
+
    ```bash
    cd web
-   npm install --force
-   ```  
-7. Konfigurasikan frontend untuk memperbarui `proxy.target` di **.umirc.ts** menjadi `http://127.0.0.1:9380`:
+   npm install
+   ```
+
 8. Jalankan aplikasi frontend:
+
    ```bash
-   npm run dev 
-   ```  
+   npm run dev
+   ```
 
    _Output berikut menandakan bahwa sistem berhasil diluncurkan:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+
+9. Hentikan layanan front-end dan back-end RAGFlow setelah pengembangan selesai:
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
 ## 📚 Dokumentasi
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [Panduan Pengguna](https://ragflow.io/docs/dev/category/guides)
-- [Referensi](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
+- [References](https://ragflow.io/docs/dev/category/references)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 Roadmap
 
-Lihat [Roadmap RAGFlow 2024](https://github.com/infiniflow/ragflow/issues/162)
+Lihat [Roadmap RAGFlow 2025](https://github.com/infiniflow/ragflow/issues/4214)
 
 ## 🏄 Komunitas
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/NjYzJD3GM3)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Kontribusi
 
 RAGFlow berkembang melalui kolaborasi open-source. Dalam semangat ini, kami menerima kontribusi dari komunitas.
-Jika Anda ingin berpartisipasi, tinjau terlebih dahulu [Panduan Kontribusi](./CONTRIBUTING.md).
+Jika Anda ingin berpartisipasi, tinjau terlebih dahulu [Panduan Kontribusi](https://ragflow.io/docs/dev/contributing).

README_ja.md

@@ -7,9 +7,11 @@
 <p align="center">
   <a href="./README.md">English</a> |
   <a href="./README_zh.md">简体中文</a> |
+  <a href="./README_tzh.md">繁体中文</a> |
   <a href="./README_ja.md">日本語</a> |
   <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a>
+  <a href="./README_id.md">Bahasa Indonesia</a> |
+  <a href="/README_pt_br.md">Português (Brasil)</a>
 </p>
 
 <p align="center">
@@ -20,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.14.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.14.0">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,12 +32,11 @@
     </a>
 </p>
 
-
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
-  <a href="https://github.com/infiniflow/ragflow/issues/162">Roadmap</a> |
+  <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
@@ -46,22 +47,25 @@
 ## 🎮 Demo
 
 デモをお試しください：[https://demo.ragflow.io](https://demo.ragflow.io)。
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/infiniflow/ragflow/assets/7248/2f6baa3e-1092-4f11-866d-36f6a9d075e5" width="1200"/>
 <img src="https://github.com/user-attachments/assets/504bbbf1-c9f7-4d83-8cc5-e9cb63c26db6" width="1200"/>
 </div>
 
-
 ## 🔥 最新情報
 
-- 2024-11-22 エージェントでの変数の定義と使用法を改善しました。
+- 2025-03-19 PDFまたはDOCXファイル内の画像を理解するために、多モーダルモデルを使用することをサポートします。
+- 2025-02-28 インターネット検索 (TAVILY) と組み合わせて、あらゆる LLM の詳細な調査をサポートします。
+- 2025-01-26 ナレッジ グラフの抽出と適用を最適化し、さまざまな構成オプションを提供します。
+- 2024-12-18 DeepDoc のドキュメント レイアウト分析モデルをアップグレードします。
 - 2024-11-01 再現の精度を向上させるために、解析されたチャンクにキーワード抽出と関連質問の生成を追加しました。
-- 2024-09-13 ナレッジベース Q&A の検索モードを追加しました。
 - 2024-08-22 RAG を介して SQL ステートメントへのテキストをサポートします。
-- 2024-08-02 [graphrag](https://github.com/microsoft/graphrag) からインスピレーションを得た GraphRAG とマインド マップをサポートします。
 
 ## 🎉 続きを楽しみに
+
 ⭐️ リポジトリをスター登録して、エキサイティングな新機能やアップデートを最新の状態に保ちましょう！すべての新しいリリースに関する即時通知を受け取れます！ 🌟
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
 </div>
@@ -108,7 +112,10 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > ローカルマシン（Windows、Mac、または Linux）に Docker をインストールしていない場合は、[Docker Engine のインストール](https://docs.docker.com/engine/install/) を参照してください。
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): RAGFlowのコード実行（サンドボックス）機能を利用する場合のみ必要です。
+
+> [!TIP]
+> ローカルマシン（Windows、Mac、または Linux）に Docker をインストールしていない場合は、[Docker Engine のインストール](https://docs.docker.com/engine/install/) を参照してください。
 
 ### 🚀 サーバーを起動
 
@@ -141,20 +148,29 @@
 
 3. ビルド済みの Docker イメージをビルドし、サーバーを起動する:
 
-   > 以下のコマンドは、RAGFlow slim（`dev-slim`）の開発版Dockerイメージをダウンロードします。RAGFlow slimのDockerイメージには、埋め込みモデルやPythonライブラリが含まれていないため、サイズは約1GBです。
+> [!CAUTION]
+> 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
+> ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
+
+   > 以下のコマンドは、RAGFlow Docker イメージの v0.19.0-slim エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.19.0-slim とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。例えば、完全版 v0.19.0 をダウンロードするには、RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0 と設定します。
 
    ```bash
    $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
    ```
 
-   > - 特定のバージョンのRAGFlow slim Dockerイメージをダウンロードするには、**docker/.env**内の`RAGFlow_IMAGE`変数を希望のバージョンに更新します。例えば、`RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0`とします。この変更を行った後、上記のコマンドを再実行してダウンロードを開始してください。
-   > - RAGFlowの埋め込みモデルとPythonライブラリを含む開発版Dockerイメージをダウンロードするには、**docker/.env**内の`RAGFlow_IMAGE`変数を`RAGFLOW_IMAGE=infiniflow/ragflow:dev`に更新します。この変更を行った後、上記のコマンドを再実行してダウンロードを開始してください。
-   > - 特定のバージョンのRAGFlow Dockerイメージ（埋め込みモデルとPythonライブラリを含む）をダウンロードするには、**docker/.env**内の`RAGFlow_IMAGE`変数を希望のバージョンに更新します。例えば、`RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0`とします。この変更を行った後、上記のコマンドを再実行してダウンロードを開始してください。  
-   
-   > **NOTE:** 埋め込みモデルとPythonライブラリを含むRAGFlow Dockerイメージのサイズは約9GBであり、読み込みにかなりの時間がかかる場合があります。
+   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+   | ----------------- | --------------- | --------------------- | ------------------------ |
+   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 
-4. サーバーを立ち上げた後、サーバーの状態を確認する:
+1. サーバーを立ち上げた後、サーバーの状態を確認する:
 
    ```bash
    $ docker logs -f ragflow-server
@@ -163,22 +179,20 @@
    _以下の出力は、システムが正常に起動したことを確認するものです:_
 
    ```bash
-        ____   ___    ______ ______ __               
+        ____   ___    ______ ______ __
        / __ \ /   |  / ____// ____// /____  _      __
       / /_/ // /| | / / __ / /_   / // __ \| | /| / /
-     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ / 
-    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/  
+     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
+    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
+
    > もし確認ステップをスキップして直接 RAGFlow にログインした場合、その時点で RAGFlow が完全に初期化されていない可能性があるため、ブラウザーがネットワーク異常エラーを表示するかもしれません。
 
-5. ウェブブラウザで、プロンプトに従ってサーバーの IP アドレスを入力し、RAGFlow にログインします。
+2. ウェブブラウザで、プロンプトに従ってサーバーの IP アドレスを入力し、RAGFlow にログインします。
    > デフォルトの設定を使用する場合、デフォルトの HTTP サービングポート `80` は省略できるので、与えられたシナリオでは、`http://IP_OF_YOUR_MACHINE`（ポート番号は省略）だけを入力すればよい。
-6. [service_conf.yaml](./docker/service_conf.yaml) で、`user_default_llm` で希望の LLM ファクトリを選択し、`API_KEY` フィールドを対応する API キーで更新する。
+3. [service_conf.yaml.template](./docker/service_conf.yaml.template) で、`user_default_llm` で希望の LLM ファクトリを選択し、`API_KEY` フィールドを対応する API キーで更新する。
 
    > 詳しくは [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) を参照してください。
 
@@ -189,19 +203,19 @@
 システムコンフィグに関しては、以下のファイルを管理する必要がある:
 
 - [.env](./docker/.env): `SVR_HTTP_PORT`、`MYSQL_PASSWORD`、`MINIO_PASSWORD` などのシステムの基本設定を保持する。
-- [service_conf.yaml](./docker/service_conf.yaml): バックエンドのサービスを設定します。
+- [service_conf.yaml.template](./docker/service_conf.yaml.template): バックエンドのサービスを設定します。
 - [docker-compose.yml](./docker/docker-compose.yml): システムの起動は [docker-compose.yml](./docker/docker-compose.yml) に依存している。
 
-[.env](./docker/.env) ファイルの変更が [service_conf.yaml](./docker/service_conf.yaml) ファイルの内容と一致していることを確認する必要があります。
+[.env](./docker/.env) ファイルの変更が [service_conf.yaml.template](./docker/service_conf.yaml.template) ファイルの内容と一致していることを確認する必要があります。
 
-> [./docker/README](./docker/README.md) ファイルは環境設定とサービスコンフィグの詳細な説明を提供し、[./docker/README](./docker/README.md) ファイルに記載されている全ての環境設定が [service_conf.yaml](./docker/service_conf.yaml) ファイルの対応するコンフィグと一致していることを確認することが義務付けられています。
+> [./docker/README](./docker/README.md) ファイル ./docker/README には、service_conf.yaml.template ファイルで ${ENV_VARS} として使用できる環境設定とサービス構成の詳細な説明が含まれています。
 
 デフォルトの HTTP サービングポート(80)を更新するには、[docker-compose.yml](./docker/docker-compose.yml) にアクセスして、`80:80` を `<YOUR_SERVING_PORT>:80` に変更します。
 
 > すべてのシステム設定のアップデートを有効にするには、システムの再起動が必要です:
 >
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ### Elasticsearch から Infinity にドキュメントエンジンを切り替えます
@@ -212,110 +226,133 @@ RAGFlow はデフォルトで Elasticsearch を使用して全文とベクトル
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
-2. **docker/.env** の「DOC _ ENGINE」を「infinity」に設定します。
+   Note: `-v` は docker コンテナのボリュームを削除し、既存のデータをクリアします。
+2. **docker/.env** の「DOC \_ ENGINE」を「infinity」に設定します。
 
 3. 起動コンテナ：
    ```bash
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
-> [!WARNING]  
-> Linux/arm64 マシンでの Infinity への切り替えは正式にサポートされていません。
+   > [!WARNING]
+   > Linux/arm64 マシンでの Infinity への切り替えは正式にサポートされていません。
 
-## 🔧 ソースコードでDockerイメージを作成（埋め込みモデルなし）
+## 🔧 ソースコードで Docker イメージを作成（埋め込みモデルなし）
 
 この Docker イメージのサイズは約 1GB で、外部の大モデルと埋め込みサービスに依存しています。
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
-## 🔧 ソースコードをコンパイルしたDockerイメージ（埋め込みモデルを含む）
+## 🔧 ソースコードをコンパイルした Docker イメージ（埋め込みモデルを含む）
 
 この Docker のサイズは約 9GB で、埋め込みモデルを含むため、外部の大モデルサービスのみが必要です。
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile -t infiniflow/ragflow:dev .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 ソースコードからサービスを起動する方法
 
-1. Poetry をインストールする。すでにインストールされている場合は、このステップをスキップしてください:
+1. uv をインストールする。すでにインストールされている場合は、このステップをスキップしてください:
+
    ```bash
-   curl -sSL https://install.python-poetry.org | python3 -
+   pipx install uv pre-commit
    ```
 
 2. ソースコードをクローンし、Python の依存関係をインストールする:
+
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true
-   ~/.local/bin/poetry install --sync --no-root # install RAGFlow dependent python modules
+   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. Docker Compose を使用して依存サービス（MinIO、Elasticsearch、Redis、MySQL）を起動する:
+
    ```bash
    docker compose -f docker/docker-compose-base.yml up -d
    ```
 
-   `/etc/hosts` に以下の行を追加して、**docker/service_conf.yaml** に指定されたすべてのホストを `127.0.0.1` に解決します:  
+   `/etc/hosts` に以下の行を追加して、**conf/service_conf.yaml** に指定されたすべてのホストを `127.0.0.1` に解決します:
+
+   ```
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-   127.0.0.1       es01 infinity mysql minio redis
-   ```  
-   **docker/service_conf.yaml** で mysql のポートを `5455` に、es のポートを `1200` に更新します（**docker/.env** に指定された通り）.
 
 4. HuggingFace にアクセスできない場合は、`HF_ENDPOINT` 環境変数を設定してミラーサイトを使用してください:
- 
+
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. バックエンドサービスを起動する:
+5. オペレーティングシステムにjemallocがない場合は、次のようにインストールします:
+   
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+
+6. バックエンドサービスを起動する:
+
    ```bash
    source .venv/bin/activate
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
 
-6. フロントエンドの依存関係をインストールする:  
+7. フロントエンドの依存関係をインストールする:
+
    ```bash
    cd web
-   npm install --force
-   ```  
-7. フロントエンドを設定し、**.umirc.ts** の `proxy.target` を `http://127.0.0.1:9380` に更新します:
-8. フロントエンドサービスを起動する:  
-   ```bash
-   npm run dev 
+   npm install
    ```
 
-   _以下の画面で、システムが正常に起動したことを示します:_  
+8. フロントエンドサービスを起動する:
+
+   ```bash
+   npm run dev
+   ```
+
+   _以下の画面で、システムが正常に起動したことを示します:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+9. 開発が完了したら、RAGFlow のフロントエンド サービスとバックエンド サービスを停止します:
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
 ## 📚 ドキュメンテーション
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/guides)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
 - [References](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 ロードマップ
 
-[RAGFlow ロードマップ 2024](https://github.com/infiniflow/ragflow/issues/162) を参照
+[RAGFlow ロードマップ 2025](https://github.com/infiniflow/ragflow/issues/4214) を参照
 
 ## 🏄 コミュニティ
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/NjYzJD3GM3)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 コントリビュート
 
-RAGFlow はオープンソースのコラボレーションによって発展してきました。この精神に基づき、私たちはコミュニティからの多様なコントリビュートを受け入れています。 参加を希望される方は、まず [コントリビューションガイド](./CONTRIBUTING.md)をご覧ください。
+RAGFlow はオープンソースのコラボレーションによって発展してきました。この精神に基づき、私たちはコミュニティからの多様なコントリビュートを受け入れています。 参加を希望される方は、まず [コントリビューションガイド](https://ragflow.io/docs/dev/contributing)をご覧ください。

README_ko.md

@@ -7,9 +7,11 @@
 <p align="center">
   <a href="./README.md">English</a> |
   <a href="./README_zh.md">简体中文</a> |
+  <a href="./README_tzh.md">繁体中文</a> |
   <a href="./README_ja.md">日本語</a> |
   <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a>
+  <a href="./README_id.md">Bahasa Indonesia</a> |
+  <a href="/README_pt_br.md">Português (Brasil)</a>
 </p>
 
 <p align="center">
@@ -20,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.14.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.14.0">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,76 +32,72 @@
     </a>
 </p>
 
-
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
-  <a href="https://github.com/infiniflow/ragflow/issues/162">Roadmap</a> |
+  <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-
 ## 💡 RAGFlow란?
 
 [RAGFlow](https://ragflow.io/)는 심층 문서 이해에 기반한 오픈소스 RAG (Retrieval-Augmented Generation) 엔진입니다. 이 엔진은 대규모 언어 모델(LLM)과 결합하여 정확한 질문 응답 기능을 제공하며, 다양한 복잡한 형식의 데이터에서 신뢰할 수 있는 출처를 바탕으로 한 인용을 통해 이를 뒷받침합니다. RAGFlow는 규모에 상관없이 모든 기업에 최적화된 RAG 워크플로우를 제공합니다.
 
-
-
 ## 🎮 데모
+
 데모를 [https://demo.ragflow.io](https://demo.ragflow.io)에서 실행해 보세요.
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/infiniflow/ragflow/assets/7248/2f6baa3e-1092-4f11-866d-36f6a9d075e5" width="1200"/>
 <img src="https://github.com/user-attachments/assets/504bbbf1-c9f7-4d83-8cc5-e9cb63c26db6" width="1200"/>
 </div>
 
-
 ## 🔥 업데이트
 
-- 2024-11-22 에이전트의 변수 정의 및 사용을 개선했습니다.
-  
+- 2025-03-19 PDF 또는 DOCX 파일 내의 이미지를 이해하기 위해 다중 모드 모델을 사용하는 것을 지원합니다.
+- 2025-02-28 인터넷 검색(TAVILY)과 결합되어 모든 LLM에 대한 심층 연구를 지원합니다.
+- 2025-01-26 지식 그래프 추출 및 적용을 최적화하고 다양한 구성 옵션을 제공합니다.
+- 2024-12-18 DeepDoc의 문서 레이아웃 분석 모델 업그레이드.
 - 2024-11-01 파싱된 청크에 키워드 추출 및 관련 질문 생성을 추가하여 재현율을 향상시킵니다.
-  
-- 2024-09-13 지식베이스 Q&A 검색 모드를 추가합니다.
-  
 - 2024-08-22 RAG를 통해 SQL 문에 텍스트를 지원합니다.
-  
-- 2024-08-02: [graphrag](https://github.com/microsoft/graphrag)와 마인드맵에서 영감을 받은 GraphRAG를 지원합니다.
-
 
 ## 🎉 계속 지켜봐 주세요
+
 ⭐️우리의 저장소를 즐겨찾기에 등록하여 흥미로운 새로운 기능과 업데이트를 최신 상태로 유지하세요! 모든 새로운 릴리스에 대한 즉시 알림을 받으세요! 🌟
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
 </div>
 
-
 ## 🌟 주요 기능
 
 ### 🍭 **"Quality in, quality out"**
+
 - [심층 문서 이해](./deepdoc/README.md)를 기반으로 복잡한 형식의 비정형 데이터에서 지식을 추출합니다.
 - 문자 그대로 무한한 토큰에서 "데이터 속의 바늘"을 찾아냅니다.
 
 ### 🍱 **템플릿 기반의 chunking**
+
 - 똑똑하고 설명 가능한 방식.
 - 다양한 템플릿 옵션을 제공합니다.
 
-
 ### 🌱 **할루시네이션을 줄인 신뢰할 수 있는 인용**
+
 - 텍스트 청킹을 시각화하여 사용자가 개입할 수 있도록 합니다.
 - 중요한 참고 자료와 추적 가능한 인용을 빠르게 확인하여 신뢰할 수 있는 답변을 지원합니다.
 
-
 ### 🍔 **다른 종류의 데이터 소스와의 호환성**
+
 - 워드, 슬라이드, 엑셀, 텍스트 파일, 이미지, 스캔본, 구조화된 데이터, 웹 페이지 등을 지원합니다.
 
 ### 🛀 **자동화되고 손쉬운 RAG 워크플로우**
+
 - 개인 및 대규모 비즈니스에 맞춘 효율적인 RAG 오케스트레이션.
 - 구성 가능한 LLM 및 임베딩 모델.
 - 다중 검색과 결합된 re-ranking.
 - 비즈니스와 원활하게 통합할 수 있는 직관적인 API.
 
-
 ## 🔎 시스템 아키텍처
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -107,17 +105,22 @@
 </div>
 
 ## 🎬 시작하기
+
 ### 📝 사전 준비 사항
+
 - CPU >= 4 cores
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > 로컬 머신(Windows, Mac, Linux)에 Docker가 설치되지 않은 경우, [Docker 엔진 설치]((https://docs.docker.com/engine/install/))를 참조하세요.
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): RAGFlow의 코드 실행기(샌드박스) 기능을 사용하려는 경우에만 필요합니다.
 
+> [!TIP]
+> 로컬 머신(Windows, Mac, Linux)에 Docker가 설치되지 않은 경우, [Docker 엔진 설치](<(https://docs.docker.com/engine/install/)>)를 참조하세요.
 
 ### 🚀 서버 시작하기
 
 1. `vm.max_map_count`가 262144 이상인지 확인하세요:
+
    > `vm.max_map_count`의 값을 아래 명령어를 통해 확인하세요:
    >
    > ```bash
@@ -145,21 +148,29 @@
 
 3. 미리 빌드된 Docker 이미지를 생성하고 서버를 시작하세요:
 
-   > 아래의 명령은 RAGFlow slim(dev-slim)의 개발 버전 Docker 이미지를 다운로드합니다. RAGFlow slim Docker 이미지에는 임베딩 모델이나 Python 라이브러리가 포함되어 있지 않으므로 크기는 약 1GB입니다.
+> [!CAUTION]
+> 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
+> ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
+
+   > 아래 명령어는 RAGFlow Docker 이미지의 v0.19.0-slim 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.19.0-slim과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오. 예를 들어, 전체 버전인 v0.19.0을 다운로드하려면 RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0로 설정합니다.
 
    ```bash
    $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
    ```
-   
-   > - 특정 버전의 RAGFlow slim Docker 이미지를 다운로드하려면, **docker/.env**에서 `RAGFlow_IMAGE` 변수를 원하는 버전으로 업데이트하세요. 예를 들어, `RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0-slim`으로 설정합니다. 이 변경을 완료한 후, 위의 명령을 다시 실행하여 다운로드를 시작하세요.
-   > - RAGFlow의 임베딩 모델과 Python 라이브러리를 포함한 개발 버전 Docker 이미지를 다운로드하려면, **docker/.env**에서 `RAGFlow_IMAGE` 변수를 `RAGFLOW_IMAGE=infiniflow/ragflow:dev`로 업데이트하세요. 이 변경을 완료한 후, 위의 명령을 다시 실행하여 다운로드를 시작하세요.
-   > - 특정 버전의 RAGFlow Docker 이미지를 임베딩 모델과 Python 라이브러리를 포함하여 다운로드하려면, **docker/.env**에서 `RAGFlow_IMAGE` 변수를 원하는 버전으로 업데이트하세요. 예를 들어, `RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0` 로 설정합니다. 이 변경을 완료한 후, 위의 명령을 다시 실행하여 다운로드를 시작하세요.  
-   
-   > **NOTE:** 임베딩 모델과 Python 라이브러리를 포함한 RAGFlow Docker 이미지의 크기는 약 9GB이며, 로드하는 데 상당히 오랜 시간이 걸릴 수 있습니다.
 
+   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+   | ----------------- | --------------- | --------------------- | ------------------------ |
+   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
 
-4. 서버가 시작된 후 서버 상태를 확인하세요:
+1. 서버가 시작된 후 서버 상태를 확인하세요:
 
    ```bash
    $ docker logs -f ragflow-server
@@ -168,22 +179,21 @@
    _다음 출력 결과로 시스템이 성공적으로 시작되었음을 확인합니다:_
 
    ```bash
-        ____   ___    ______ ______ __               
+        ____   ___    ______ ______ __
        / __ \ /   |  / ____// ____// /____  _      __
       / /_/ // /| | / / __ / /_   / // __ \| | /| / /
-     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ / 
-    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/  
+     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
+    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
+
    > 만약 확인 단계를 건너뛰고 바로 RAGFlow에 로그인하면, RAGFlow가 완전히 초기화되지 않았기 때문에 브라우저에서 `network anormal` 오류가 발생할 수 있습니다.
 
-5. 웹 브라우저에 서버의 IP 주소를 입력하고 RAGFlow에 로그인하세요.
+2. 웹 브라우저에 서버의 IP 주소를 입력하고 RAGFlow에 로그인하세요.
    > 기본 설정을 사용할 경우, `http://IP_OF_YOUR_MACHINE`만 입력하면 됩니다 (포트 번호는 제외). 기본 HTTP 서비스 포트 `80`은 기본 구성으로 사용할 때 생략할 수 있습니다.
-6. [service_conf.yaml](./docker/service_conf.yaml) 파일에서 원하는 LLM 팩토리를 `user_default_llm`에 선택하고, `API_KEY` 필드를 해당 API 키로 업데이트하세요.
+3. [service_conf.yaml.template](./docker/service_conf.yaml.template) 파일에서 원하는 LLM 팩토리를 `user_default_llm`에 선택하고, `API_KEY` 필드를 해당 API 키로 업데이트하세요.
+
    > 자세한 내용은 [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup)를 참조하세요.
 
    _이제 쇼가 시작됩니다!_
@@ -193,36 +203,38 @@
 시스템 설정과 관련하여 다음 파일들을 관리해야 합니다:
 
 - [.env](./docker/.env): `SVR_HTTP_PORT`, `MYSQL_PASSWORD`, `MINIO_PASSWORD`와 같은 시스템의 기본 설정을 포함합니다.
-- [service_conf.yaml](./docker/service_conf.yaml): 백엔드 서비스를 구성합니다.
+- [service_conf.yaml.template](./docker/service_conf.yaml.template): 백엔드 서비스를 구성합니다.
 - [docker-compose.yml](./docker/docker-compose.yml): 시스템은 [docker-compose.yml](./docker/docker-compose.yml)을 사용하여 시작됩니다.
 
-[.env](./docker/.env) 파일의 변경 사항이 [service_conf.yaml](./docker/service_conf.yaml) 파일의 내용과 일치하도록 해야 합니다.
+[.env](./docker/.env) 파일의 변경 사항이 [service_conf.yaml.template](./docker/service_conf.yaml.template) 파일의 내용과 일치하도록 해야 합니다.
 
-> [./docker/README](./docker/README.md) 파일에는 환경 설정과 서비스 구성에 대한 자세한 설명이 있으며, [./docker/README](./docker/README.md) 파일에 나열된 모든 환경 설정이 [service_conf.yaml](./docker/service_conf.yaml) 파일의 해당 구성과 일치하도록 해야 합니다.
+> [./docker/README](./docker/README.md) 파일 ./docker/README은 service_conf.yaml.template 파일에서 ${ENV_VARS}로 사용할 수 있는 환경 설정과 서비스 구성에 대한 자세한 설명을 제공합니다.
 
 기본 HTTP 서비스 포트(80)를 업데이트하려면 [docker-compose.yml](./docker/docker-compose.yml) 파일에서 `80:80`을 `<YOUR_SERVING_PORT>:80`으로 변경하세요.
 
 > 모든 시스템 구성 업데이트는 적용되기 위해 시스템 재부팅이 필요합니다.
 >
 > ```bash
-> $ docker compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker-compose.yml up -d
 > ```
 
 ### Elasticsearch 에서 Infinity 로 문서 엔진 전환
 
 RAGFlow 는 기본적으로 Elasticsearch 를 사용하여 전체 텍스트 및 벡터를 저장합니다. [Infinity]로 전환(https://github.com/infiniflow/infinity/), 다음 절차를 따르십시오.
+
 1. 실행 중인 모든 컨테이너를 중지합니다.
    ```bash
-   $docker compose-f docker/docker-compose.yml down-v
+   $docker compose-f docker/docker-compose.yml down -v
    ```
+   Note: `-v` 는 docker 컨테이너의 볼륨을 삭제하고 기존 데이터를 지우며, 이 작업은 컨테이너를 중지하는 것과 동일합니다.
 2. **docker/.env**의 "DOC_ENGINE" 을 "infinity" 로 설정합니다.
 3. 컨테이너 부팅:
    ```bash
-   $docker compose-f docker/docker-compose.yml up-d
-   ``` 
-> [!WARNING]
-> Linux/arm64 시스템에서 Infinity로 전환하는 것은 공식적으로 지원되지 않습니다.
- 
+   $docker compose-f docker/docker-compose.yml up -d
+   ```
+   > [!WARNING]
+   > Linux/arm64 시스템에서 Infinity로 전환하는 것은 공식적으로 지원되지 않습니다.
+
 ## 🔧 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함하지 않음)
 
 이 Docker 이미지의 크기는 약 1GB이며, 외부 대형 모델과 임베딩 서비스에 의존합니다.
@@ -230,9 +242,7 @@ RAGFlow 는 기본적으로 Elasticsearch 를 사용하여 전체 텍스트 및
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함)
@@ -242,82 +252,107 @@ docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile -t infiniflow/ragflow:dev .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 소스 코드로 서비스를 시작합니다.
 
-1. Poetry를 설치하거나 이미 설치된 경우 이 단계를 건너뜁니다:
+1. uv를 설치하거나 이미 설치된 경우 이 단계를 건너뜁니다:
+
    ```bash
-   curl -sSL https://install.python-poetry.org | python3 -
+   pipx install uv pre-commit
    ```
 
 2. 소스 코드를 클론하고 Python 의존성을 설치합니다:
+
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true
-   ~/.local/bin/poetry install --sync --no-root # install RAGFlow dependent python modules
+   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
 3. Docker Compose를 사용하여 의존 서비스(MinIO, Elasticsearch, Redis 및 MySQL)를 시작합니다:
+
    ```bash
    docker compose -f docker/docker-compose-base.yml up -d
    ```
 
-   `/etc/hosts` 에 다음 줄을 추가하여 **docker/service_conf.yaml** 에 지정된 모든 호스트를 `127.0.0.1` 로 해결합니다:  
+   `/etc/hosts` 에 다음 줄을 추가하여 **conf/service_conf.yaml** 에 지정된 모든 호스트를 `127.0.0.1` 로 해결합니다:
+
+   ```
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-   127.0.0.1       es01 infinity mysql minio redis
-   ```  
-   **docker/service_conf.yaml** 에서 mysql 포트를 `5455` 로, es 포트를 `1200` 으로 업데이트합니다( **docker/.env** 에 지정된 대로).
 
 4. HuggingFace에 접근할 수 없는 경우, `HF_ENDPOINT` 환경 변수를 설정하여 미러 사이트를 사용하세요:
- 
+
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. 백엔드 서비스를 시작합니다:
+5. 만약 운영 체제에 jemalloc이 없으면 다음 방식으로 설치하세요:
+
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+
+6. 백엔드 서비스를 시작합니다:
+
    ```bash
    source .venv/bin/activate
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
 
-6. 프론트엔드 의존성을 설치합니다:  
+7. 프론트엔드 의존성을 설치합니다:
+
    ```bash
    cd web
-   npm install --force
-   ```  
-7. **.umirc.ts** 에서 `proxy.target` 을 `http://127.0.0.1:9380` 으로 업데이트합니다:
-8. 프론트엔드 서비스를 시작합니다:  
-   ```bash
-   npm run dev 
+   npm install
    ```
 
-   _다음 인터페이스는 시스템이 성공적으로 시작되었음을 나타냅니다:_  
+8. 프론트엔드 서비스를 시작합니다:
+
+   ```bash
+   npm run dev
+   ```
+
+   _다음 인터페이스는 시스템이 성공적으로 시작되었음을 나타냅니다:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+
+9. 개발이 완료된 후 RAGFlow 프론트엔드 및 백엔드 서비스를 중지합니다.
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+   
+
 ## 📚 문서
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/guides)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
 - [References](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 로드맵
 
-[RAGFlow 로드맵 2024](https://github.com/infiniflow/ragflow/issues/162)을 확인하세요.
+[RAGFlow 로드맵 2025](https://github.com/infiniflow/ragflow/issues/4214)을 확인하세요.
 
 ## 🏄 커뮤니티
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/NjYzJD3GM3)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 컨트리뷰션
 
-RAGFlow는 오픈소스 협업을 통해 발전합니다. 이러한 정신을 바탕으로, 우리는 커뮤니티의 다양한 기여를 환영합니다. 참여하고 싶으시다면, 먼저 [가이드라인](./CONTRIBUTING.md)을 검토해 주세요.
+RAGFlow는 오픈소스 협업을 통해 발전합니다. 이러한 정신을 바탕으로, 우리는 커뮤니티의 다양한 기여를 환영합니다. 참여하고 싶으시다면, 먼저 [가이드라인](https://ragflow.io/docs/dev/contributing)을 검토해 주세요.

README_pt_br.md

@@ -0,0 +1,382 @@
+<div align="center">
+<a href="https://demo.ragflow.io/">
+<img src="web/src/assets/logo-with-text.png" width="520" alt="ragflow logo">
+</a>
+</div>
+
+<p align="center">
+  <a href="./README.md">English</a> |
+  <a href="./README_zh.md">简体中文</a> |
+  <a href="./README_tzh.md">繁体中文</a> |
+  <a href="./README_ja.md">日本語</a> |
+  <a href="./README_ko.md">한국어</a> |
+  <a href="./README_id.md">Bahasa Indonesia</a> |
+  <a href="/README_pt_br.md">Português (Brasil)</a>
+</p>
+
+<p align="center">
+    <a href="https://x.com/intent/follow?screen_name=infiniflowai" target="_blank">
+        <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="seguir no X(Twitter)">
+    </a>
+    <a href="https://demo.ragflow.io" target="_blank">
+        <img alt="Badge Estático" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+    </a>
+    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
+    </a>
+    <a href="https://github.com/infiniflow/ragflow/releases/latest">
+        <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Última%20Relese" alt="Última Versão">
+    </a>
+    <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
+        <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="licença">
+    </a>
+</p>
+
+<h4 align="center">
+  <a href="https://ragflow.io/docs/dev/">Documentação</a> |
+  <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
+  <a href="https://twitter.com/infiniflowai">Twitter</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
+  <a href="https://demo.ragflow.io">Demo</a>
+</h4>
+
+<details open>
+<summary><b>📕 Índice</b></summary>
+
+- 💡 [O que é o RAGFlow?](#-o-que-é-o-ragflow)
+- 🎮 [Demo](#-demo)
+- 📌 [Últimas Atualizações](#-últimas-atualizações)
+- 🌟 [Principais Funcionalidades](#-principais-funcionalidades)
+- 🔎 [Arquitetura do Sistema](#-arquitetura-do-sistema)
+- 🎬 [Primeiros Passos](#-primeiros-passos)
+- 🔧 [Configurações](#-configurações)
+- 🔧 [Construir uma imagem docker sem incorporar modelos](#-construir-uma-imagem-docker-sem-incorporar-modelos)
+- 🔧 [Construir uma imagem docker incluindo modelos](#-construir-uma-imagem-docker-incluindo-modelos)
+- 🔨 [Lançar serviço a partir do código-fonte para desenvolvimento](#-lançar-serviço-a-partir-do-código-fonte-para-desenvolvimento)
+- 📚 [Documentação](#-documentação)
+- 📜 [Roadmap](#-roadmap)
+- 🏄 [Comunidade](#-comunidade)
+- 🙌 [Contribuindo](#-contribuindo)
+
+</details>
+
+## 💡 O que é o RAGFlow?
+
+[RAGFlow](https://ragflow.io/) é um mecanismo RAG (Geração Aumentada por Recuperação) de código aberto baseado em entendimento profundo de documentos. Ele oferece um fluxo de trabalho RAG simplificado para empresas de qualquer porte, combinando LLMs (Modelos de Linguagem de Grande Escala) para fornecer capacidades de perguntas e respostas verídicas, respaldadas por citações bem fundamentadas de diversos dados complexos formatados.
+
+## 🎮 Demo
+
+Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
+
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/infiniflow/ragflow/assets/7248/2f6baa3e-1092-4f11-866d-36f6a9d075e5" width="1200"/>
+<img src="https://github.com/user-attachments/assets/504bbbf1-c9f7-4d83-8cc5-e9cb63c26db6" width="1200"/>
+</div>
+
+## 🔥 Últimas Atualizações
+
+- 19-03-2025 Suporta o uso de um modelo multi-modal para entender imagens dentro de arquivos PDF ou DOCX.
+- 28-02-2025 combinado com a pesquisa na Internet (T AVI LY), suporta pesquisas profundas para qualquer LLM.
+- 26-01-2025 Otimize a extração e aplicação de gráficos de conhecimento e forneça uma variedade de opções de configuração.
+- 18-12-2024 Atualiza o modelo de Análise de Layout de Documentos no DeepDoc.
+- 01-11-2024 Adiciona extração de palavras-chave e geração de perguntas relacionadas aos blocos analisados para melhorar a precisão da recuperação.
+- 22-08-2024 Suporta conversão de texto para comandos SQL via RAG.
+
+## 🎉 Fique Ligado
+
+⭐️ Dê uma estrela no nosso repositório para se manter atualizado com novas funcionalidades e melhorias empolgantes! Receba notificações instantâneas sobre novos lançamentos! 🌟
+
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
+</div>
+
+## 🌟 Principais Funcionalidades
+
+### 🍭 **"Qualidade entra, qualidade sai"**
+
+- Extração de conhecimento baseada em [entendimento profundo de documentos](./deepdoc/README.md) a partir de dados não estruturados com formatos complicados.
+- Encontra a "agulha no palheiro de dados" de literalmente tokens ilimitados.
+
+### 🍱 **Fragmentação baseada em templates**
+
+- Inteligente e explicável.
+- Muitas opções de templates para escolher.
+
+### 🌱 **Citações fundamentadas com menos alucinações**
+
+- Visualização da fragmentação de texto para permitir intervenção humana.
+- Visualização rápida das referências chave e citações rastreáveis para apoiar respostas fundamentadas.
+
+### 🍔 **Compatibilidade com fontes de dados heterogêneas**
+
+- Suporta Word, apresentações, excel, txt, imagens, cópias digitalizadas, dados estruturados, páginas da web e mais.
+
+### 🛀 **Fluxo de trabalho RAG automatizado e sem esforço**
+
+- Orquestração RAG simplificada voltada tanto para negócios pessoais quanto grandes empresas.
+- Modelos LLM e de incorporação configuráveis.
+- Múltiplas recuperações emparelhadas com reclassificação fundida.
+- APIs intuitivas para integração sem problemas com os negócios.
+
+## 🔎 Arquitetura do Sistema
+
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+</div>
+
+## 🎬 Primeiros Passos
+
+### 📝 Pré-requisitos
+
+- CPU >= 4 núcleos
+- RAM >= 16 GB
+- Disco >= 50 GB
+- Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): Necessário apenas se você pretende usar o recurso de executor de código (sandbox) do RAGFlow.
+
+> [!TIP]
+> Se você não instalou o Docker na sua máquina local (Windows, Mac ou Linux), veja [Instalar Docker Engine](https://docs.docker.com/engine/install/).
+
+### 🚀 Iniciar o servidor
+
+1.  Certifique-se de que `vm.max_map_count` >= 262144:
+
+    > Para verificar o valor de `vm.max_map_count`:
+    >
+    > ```bash
+    > $ sysctl vm.max_map_count
+    > ```
+    >
+    > Se necessário, redefina `vm.max_map_count` para um valor de pelo menos 262144:
+    >
+    > ```bash
+    > # Neste caso, defina para 262144:
+    > $ sudo sysctl -w vm.max_map_count=262144
+    > ```
+    >
+    > Essa mudança será resetada após a reinicialização do sistema. Para garantir que a alteração permaneça permanente, adicione ou atualize o valor de `vm.max_map_count` em **/etc/sysctl.conf**:
+    >
+    > ```bash
+    > vm.max_map_count=262144
+    > ```
+
+2.  Clone o repositório:
+
+    ```bash
+    $ git clone https://github.com/infiniflow/ragflow.git
+    ```
+
+3.  Inicie o servidor usando as imagens Docker pré-compiladas:
+
+> [!CAUTION]
+> Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
+> Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
+
+    > O comando abaixo baixa a edição `v0.19.0-slim` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.19.0-slim`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor. Por exemplo: defina `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0` para a edição completa `v0.19.0`.
+
+    ```bash
+    $ cd ragflow/docker
+    # Use CPU for embedding and DeepDoc tasks:
+    $ docker compose -f docker-compose.yml up -d
+
+    # To use GPU to accelerate embedding and DeepDoc tasks:
+    # docker compose -f docker-compose-gpu.yml up -d
+    ```
+
+    | Tag da imagem RAGFlow | Tamanho da imagem (GB) | Possui modelos de incorporação? | Estável?                 |
+    | --------------------- | ---------------------- | ------------------------------- | ------------------------ |
+    | v0.19.0               | ~9                     | :heavy_check_mark:              | Lançamento estável       |
+    | v0.19.0-slim          | ~2                     | ❌                              | Lançamento estável       |
+    | nightly               | ~9                     | :heavy_check_mark:              | _Instável_ build noturno |
+    | nightly-slim          | ~2                     | ❌                               | _Instável_ build noturno |
+
+4.  Verifique o status do servidor após tê-lo iniciado:
+
+    ```bash
+    $ docker logs -f ragflow-server
+    ```
+
+    _O seguinte resultado confirma o lançamento bem-sucedido do sistema:_
+
+    ```bash
+         ____   ___    ______ ______ __
+        / __ \ /   |  / ____// ____// /____  _      __
+       / /_/ // /| | / / __ / /_   / // __ \| | /| / /
+      / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
+     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
+
+     * Rodando em todos os endereços (0.0.0.0)
+    ```
+
+    > Se você pular essa etapa de confirmação e acessar diretamente o RAGFlow, seu navegador pode exibir um erro `network anormal`, pois, nesse momento, seu RAGFlow pode não estar totalmente inicializado.
+
+5.  No seu navegador, insira o endereço IP do seu servidor e faça login no RAGFlow.
+
+    > Com as configurações padrão, você só precisa digitar `http://IP_DO_SEU_MÁQUINA` (**sem** o número da porta), pois a porta HTTP padrão `80` pode ser omitida ao usar as configurações padrão.
+
+6.  Em [service_conf.yaml.template](./docker/service_conf.yaml.template), selecione a fábrica LLM desejada em `user_default_llm` e atualize o campo `API_KEY` com a chave de API correspondente.
+
+    > Consulte [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) para mais informações.
+
+_O show está no ar!_
+
+## 🔧 Configurações
+
+Quando se trata de configurações do sistema, você precisará gerenciar os seguintes arquivos:
+
+- [.env](./docker/.env): Contém as configurações fundamentais para o sistema, como `SVR_HTTP_PORT`, `MYSQL_PASSWORD` e `MINIO_PASSWORD`.
+- [service_conf.yaml.template](./docker/service_conf.yaml.template): Configura os serviços de back-end. As variáveis de ambiente neste arquivo serão automaticamente preenchidas quando o contêiner Docker for iniciado. Quaisquer variáveis de ambiente definidas dentro do contêiner Docker estarão disponíveis para uso, permitindo personalizar o comportamento do serviço com base no ambiente de implantação.
+- [docker-compose.yml](./docker/docker-compose.yml): O sistema depende do [docker-compose.yml](./docker/docker-compose.yml) para iniciar.
+
+> O arquivo [./docker/README](./docker/README.md) fornece uma descrição detalhada das configurações do ambiente e dos serviços, que podem ser usadas como `${ENV_VARS}` no arquivo [service_conf.yaml.template](./docker/service_conf.yaml.template).
+
+Para atualizar a porta HTTP de serviço padrão (80), vá até [docker-compose.yml](./docker/docker-compose.yml) e altere `80:80` para `<SUA_PORTA_DE_SERVIÇO>:80`.
+
+Atualizações nas configurações acima exigem um reinício de todos os contêineres para que tenham efeito:
+
+> ```bash
+> $ docker compose -f docker-compose.yml up -d
+> ```
+
+### Mudar o mecanismo de documentos de Elasticsearch para Infinity
+
+O RAGFlow usa o Elasticsearch por padrão para armazenar texto completo e vetores. Para mudar para o [Infinity](https://github.com/infiniflow/infinity/), siga estas etapas:
+
+1. Pare todos os contêineres em execução:
+
+   ```bash
+   $ docker compose -f docker/docker-compose.yml down -v
+   ```
+   Note: `-v` irá deletar os volumes do contêiner, e os dados existentes serão apagados.
+2. Defina `DOC_ENGINE` no **docker/.env** para `infinity`.
+
+3. Inicie os contêineres:
+
+   ```bash
+   $ docker compose -f docker-compose.yml up -d
+   ```
+
+> [!ATENÇÃO]
+> A mudança para o Infinity em uma máquina Linux/arm64 ainda não é oficialmente suportada.
+
+## 🔧 Criar uma imagem Docker sem modelos de incorporação
+
+Esta imagem tem cerca de 2 GB de tamanho e depende de serviços externos de LLM e incorporação.
+
+```bash
+git clone https://github.com/infiniflow/ragflow.git
+cd ragflow/
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+```
+
+## 🔧 Criar uma imagem Docker incluindo modelos de incorporação
+
+Esta imagem tem cerca de 9 GB de tamanho. Como inclui modelos de incorporação, depende apenas de serviços externos de LLM.
+
+```bash
+git clone https://github.com/infiniflow/ragflow.git
+cd ragflow/
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
+```
+
+## 🔨 Lançar o serviço a partir do código-fonte para desenvolvimento
+
+1. Instale o `uv`, ou pule esta etapa se ele já estiver instalado:
+
+   ```bash
+   pipx install uv pre-commit
+   ```
+
+2. Clone o código-fonte e instale as dependências Python:
+
+   ```bash
+   git clone https://github.com/infiniflow/ragflow.git
+   cd ragflow/
+   uv sync --python 3.10 --all-extras # instala os módulos Python dependentes do RAGFlow
+   uv run download_deps.py
+   pre-commit install
+   ```
+
+3. Inicie os serviços dependentes (MinIO, Elasticsearch, Redis e MySQL) usando Docker Compose:
+
+   ```bash
+   docker compose -f docker/docker-compose-base.yml up -d
+   ```
+
+   Adicione a seguinte linha ao arquivo `/etc/hosts` para resolver todos os hosts especificados em **docker/.env** para `127.0.0.1`:
+
+   ```
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
+   ```
+
+4. Se não conseguir acessar o HuggingFace, defina a variável de ambiente `HF_ENDPOINT` para usar um site espelho:
+
+   ```bash
+   export HF_ENDPOINT=https://hf-mirror.com
+   ```
+
+5. Se o seu sistema operacional não tiver jemalloc, instale-o da seguinte maneira:
+
+    ```bash
+    # ubuntu
+    sudo apt-get install libjemalloc-dev
+    # centos
+    sudo yum instalar jemalloc
+    ```
+
+6. Lance o serviço de back-end:
+
+   ```bash
+   source .venv/bin/activate
+   export PYTHONPATH=$(pwd)
+   bash docker/launch_backend_service.sh
+   ```
+
+7. Instale as dependências do front-end:
+
+   ```bash
+   cd web
+   npm install
+   ```
+
+8. Lance o serviço de front-end:
+
+   ```bash
+   npm run dev
+   ```
+
+   _O seguinte resultado confirma o lançamento bem-sucedido do sistema:_
+
+   ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
+
+9. Pare os serviços de front-end e back-end do RAGFlow após a conclusão do desenvolvimento:
+
+    ```bash
+    pkill -f "ragflow_server.py|task_executor.py"
+    ```
+
+
+## 📚 Documentação
+
+- [Quickstart](https://ragflow.io/docs/dev/)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
+- [References](https://ragflow.io/docs/dev/category/references)
+- [FAQs](https://ragflow.io/docs/dev/faq)
+
+## 📜 Roadmap
+
+Veja o [RAGFlow Roadmap 2025](https://github.com/infiniflow/ragflow/issues/4214)
+
+## 🏄 Comunidade
+
+- [Discord](https://discord.gg/NjYzJD3GM3)
+- [Twitter](https://twitter.com/infiniflowai)
+- [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
+
+## 🙌 Contribuindo
+
+O RAGFlow prospera por meio da colaboração de código aberto. Com esse espírito, abraçamos contribuições diversas da comunidade.
+Se você deseja fazer parte, primeiro revise nossas [Diretrizes de Contribuição](https://ragflow.io/docs/dev/contributing).

README_tzh.md

@@ -0,0 +1,383 @@
+<div align="center">
+<a href="https://demo.ragflow.io/">
+<img src="web/src/assets/logo-with-text.png" width="350" alt="ragflow logo">
+</a>
+</div>
+
+<p align="center">
+  <a href="./README.md">English</a> |
+  <a href="./README_zh.md">简体中文</a> |
+  <a href="./README_ja.md">日本語</a> |
+  <a href="./README_ko.md">한국어</a> |
+  <a href="./README_id.md">Bahasa Indonesia</a> |
+  <a href="/README_pt_br.md">Português (Brasil)</a>
+</p>
+
+<p align="center">
+    <a href="https://x.com/intent/follow?screen_name=infiniflowai" target="_blank">
+        <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
+    </a>
+    <a href="https://demo.ragflow.io" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+    </a>
+    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
+    </a>
+    <a href="https://github.com/infiniflow/ragflow/releases/latest">
+        <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
+    </a>
+    <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
+        <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
+    </a>
+</p>
+
+<h4 align="center">
+  <a href="https://ragflow.io/docs/dev/">Document</a> |
+  <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
+  <a href="https://twitter.com/infiniflowai">Twitter</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
+  <a href="https://demo.ragflow.io">Demo</a>
+</h4>
+
+## 💡 RAGFlow 是什麼？
+
+[RAGFlow](https://ragflow.io/) 是一款基於深度文件理解所建構的開源 RAG（Retrieval-Augmented Generation）引擎。 RAGFlow 可以為各種規模的企業及個人提供一套精簡的 RAG 工作流程，結合大語言模型（LLM）針對用戶各類不同的複雜格式數據提供可靠的問答以及有理有據的引用。
+
+## 🎮 Demo 試用
+
+請登入網址 [https://demo.ragflow.io](https://demo.ragflow.io) 試用 demo。
+
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/infiniflow/ragflow/assets/7248/2f6baa3e-1092-4f11-866d-36f6a9d075e5" width="1200"/>
+<img src="https://github.com/user-attachments/assets/504bbbf1-c9f7-4d83-8cc5-e9cb63c26db6" width="1200"/>
+</div>
+
+## 🔥 近期更新
+
+- 2025-03-19 PDF和DOCX中的圖支持用多模態大模型去解析得到描述.
+- 2025-02-28 結合網路搜尋（Tavily），對於任意大模型實現類似 Deep Research 的推理功能.
+- 2025-01-26 最佳化知識圖譜的擷取與應用，提供了多種配置選擇。
+- 2024-12-18 升級了 DeepDoc 的文檔佈局分析模型。
+- 2024-11-01 對解析後的 chunk 加入關鍵字抽取和相關問題產生以提高回想的準確度。
+- 2024-08-22 支援用 RAG 技術實現從自然語言到 SQL 語句的轉換。
+
+## 🎉 關注項目
+
+⭐️ 點擊右上角的 Star 追蹤 RAGFlow，可以取得最新發布的即時通知 !🌟
+
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
+</div>
+
+## 🌟 主要功能
+
+### 🍭 **"Quality in, quality out"**
+
+- 基於[深度文件理解](./deepdoc/README.md)，能夠從各類複雜格式的非結構化資料中提取真知灼見。
+- 真正在無限上下文（token）的場景下快速完成大海撈針測試。
+
+### 🍱 **基於模板的文字切片**
+
+- 不只是智能，更重要的是可控可解釋。
+- 多種文字範本可供選擇
+
+### 🌱 **有理有據、最大程度降低幻覺（hallucination）**
+
+- 文字切片過程視覺化，支援手動調整。
+- 有理有據：答案提供關鍵引用的快照並支持追根溯源。
+
+### 🍔 **相容各類異質資料來源**
+
+- 支援豐富的文件類型，包括 Word 文件、PPT、excel 表格、txt 檔案、圖片、PDF、影印件、影印件、結構化資料、網頁等。
+
+### 🛀 **全程無憂、自動化的 RAG 工作流程**
+
+- 全面優化的 RAG 工作流程可以支援從個人應用乃至超大型企業的各類生態系統。
+- 大語言模型 LLM 以及向量模型皆支援配置。
+- 基於多路召回、融合重排序。
+- 提供易用的 API，可輕鬆整合到各類企業系統。
+
+## 🔎 系統架構
+
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+</div>
+
+## 🎬 快速開始
+
+### 📝 前提條件
+
+- CPU >= 4 核
+- RAM >= 16 GB
+- Disk >= 50 GB
+- Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): 僅在您打算使用 RAGFlow 的代碼執行器（沙箱）功能時才需要安裝。
+
+> [!TIP]
+> 如果你並沒有在本機安裝 Docker（Windows、Mac，或 Linux）, 可以參考文件 [Install Docker Engine](https://docs.docker.com/engine/install/) 自行安裝。
+
+### 🚀 啟動伺服器
+
+1. 確保 `vm.max_map_count` 不小於 262144：
+
+   > 如需確認 `vm.max_map_count` 的大小：
+   >
+   > ```bash
+   > $ sysctl vm.max_map_count
+   > ```
+   >
+   > 如果 `vm.max_map_count` 的值小於 262144，可以進行重設：
+   >
+   > ```bash
+   > # 這裡我們設為 262144:
+   > $ sudo sysctl -w vm.max_map_count=262144
+   > ```
+   >
+   > 你的改動會在下次系統重新啟動時被重置。如果希望做永久改動，還需要在 **/etc/sysctl.conf** 檔案裡把 `vm.max_map_count` 的值再相應更新一遍：
+   >
+   > ```bash
+   > vm.max_map_count=262144
+   > ```
+
+2. 克隆倉庫：
+
+   ```bash
+   $ git clone https://github.com/infiniflow/ragflow.git
+   ```
+
+3. 進入 **docker** 資料夾，利用事先編譯好的 Docker 映像啟動伺服器：
+
+> [!CAUTION]
+> 所有 Docker 映像檔都是為 x86 平台建置的。目前，我們不提供 ARM64 平台的 Docker 映像檔。
+> 如果您使用的是 ARM64 平台，請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
+
+   > 執行以下指令會自動下載 RAGFlow slim Docker 映像 `v0.19.0-slim`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.19.0-slim` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。例如，你可以透過設定 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0` 來下載 RAGFlow 鏡像的 `v0.19.0` 完整發行版。
+
+   ```bash
+   $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
+   ```
+
+   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+   | ----------------- | --------------- | --------------------- | ------------------------ |
+   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+
+   > [!TIP]
+   > 如果你遇到 Docker 映像檔拉不下來的問題，可以在 **docker/.env** 檔案內根據變數 `RAGFLOW_IMAGE` 的註解提示選擇華為雲或阿里雲的對應映像。
+   >
+   > - 華為雲鏡像名：`swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow`
+   > - 阿里雲鏡像名：`registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow`
+
+4. 伺服器啟動成功後再次確認伺服器狀態：
+
+   ```bash
+   $ docker logs -f ragflow-server
+   ```
+
+   _出現以下介面提示說明伺服器啟動成功：_
+
+   ```bash
+        ____   ___    ______ ______ __
+       / __ \ /   |  / ____// ____// /____  _      __
+      / /_/ // /| | / / __ / /_   / // __ \| | /| / /
+     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
+    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
+
+    * Running on all addresses (0.0.0.0)
+   ```
+
+   > 如果您跳過這一步驟系統確認步驟就登入 RAGFlow，你的瀏覽器有可能會提示 `network anormal` 或 `網路異常`，因為 RAGFlow 可能並未完全啟動成功。
+
+5. 在你的瀏覽器中輸入你的伺服器對應的 IP 位址並登入 RAGFlow。
+   > 上面這個範例中，您只需輸入 http://IP_OF_YOUR_MACHINE 即可：未改動過設定則無需輸入連接埠（預設的 HTTP 服務連接埠 80）。
+6. 在 [service_conf.yaml.template](./docker/service_conf.yaml.template) 檔案的 `user_default_llm` 欄位設定 LLM factory，並在 `API_KEY` 欄填入和你選擇的大模型相對應的 API key。
+
+   > 詳見 [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup)。
+
+   _好戲開始，接著奏樂接著舞！ _
+
+## 🔧 系統配置
+
+系統配置涉及以下三份文件：
+
+- [.env](./docker/.env)：存放一些系統環境變量，例如 `SVR_HTTP_PORT`、`MYSQL_PASSWORD`、`MINIO_PASSWORD` 等。
+- [service_conf.yaml.template](./docker/service_conf.yaml.template)：設定各類別後台服務。
+- [docker-compose.yml](./docker/docker-compose.yml): 系統依賴該檔案完成啟動。
+
+請務必確保 [.env](./docker/.env) 檔案中的變數設定與 [service_conf.yaml.template](./docker/service_conf.yaml.template) 檔案中的設定保持一致！
+
+如果無法存取映像網站 hub.docker.com 或模型網站 huggingface.co，請依照 [.env](./docker/.env) 註解修改 `RAGFLOW_IMAGE` 和 `HF_ENDPOINT`。
+
+> [./docker/README](./docker/README.md) 解釋了 [service_conf.yaml.template](./docker/service_conf.yaml.template) 用到的環境變數設定和服務配置。
+
+如需更新預設的 HTTP 服務連接埠(80), 可以在[docker-compose.yml](./docker/docker-compose.yml) 檔案中將配置`80:80` 改為`<YOUR_SERVING_PORT>:80` 。
+
+> 所有系統配置都需要透過系統重新啟動生效：
+>
+> ```bash
+> $ docker compose -f docker-compose.yml up -d
+> ```
+
+###把文檔引擎從 Elasticsearch 切換成為 Infinity
+
+RAGFlow 預設使用 Elasticsearch 儲存文字和向量資料. 如果要切換為 [Infinity](https://github.com/infiniflow/infinity/), 可以按照下面步驟進行:
+
+1. 停止所有容器運作:
+
+   ```bash
+   $ docker compose -f docker/docker-compose.yml down -v
+   ```
+   Note: `-v` 將會刪除 docker 容器的 volumes，已有的資料會被清空。
+
+2. 設定 **docker/.env** 目錄中的 `DOC_ENGINE` 為 `infinity`.
+
+3. 啟動容器:
+
+   ```bash
+   $ docker compose -f docker-compose.yml up -d
+   ```
+
+> [!WARNING]
+> Infinity 目前官方並未正式支援在 Linux/arm64 架構下的機器上運行.
+
+## 🔧 原始碼編譯 Docker 映像（不含 embedding 模型）
+
+本 Docker 映像大小約 2 GB 左右並且依賴外部的大模型和 embedding 服務。
+
+```bash
+git clone https://github.com/infiniflow/ragflow.git
+cd ragflow/
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+```
+
+## 🔧 原始碼編譯 Docker 映像（包含 embedding 模型）
+
+本 Docker 大小約 9 GB 左右。由於已包含 embedding 模型，所以只需依賴外部的大模型服務即可。
+
+```bash
+git clone https://github.com/infiniflow/ragflow.git
+cd ragflow/
+docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
+```
+
+## 🔨 以原始碼啟動服務
+
+1. 安裝 uv。如已安裝，可跳過此步驟：
+
+   ```bash
+   pipx install uv pre-commit
+   export UV_INDEX=https://mirrors.aliyun.com/pypi/simple
+   ```
+
+2. 下載原始碼並安裝 Python 依賴：
+
+   ```bash
+   git clone https://github.com/infiniflow/ragflow.git
+   cd ragflow/
+   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
+   ```
+
+3. 透過 Docker Compose 啟動依賴的服務（MinIO, Elasticsearch, Redis, and MySQL）：
+
+   ```bash
+   docker compose -f docker/docker-compose-base.yml up -d
+   ```
+
+   在 `/etc/hosts` 中加入以下程式碼，將 **conf/service_conf.yaml** 檔案中的所有 host 位址都解析為 `127.0.0.1`：
+
+   ```
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
+   ```
+
+4. 如果無法存取 HuggingFace，可以把環境變數 `HF_ENDPOINT` 設為對應的鏡像網站：
+
+   ```bash
+   export HF_ENDPOINT=https://hf-mirror.com
+   ```
+
+5. 如果你的操作系统没有 jemalloc，请按照如下方式安装：
+
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+
+6. 啟動後端服務：
+
+   ```bash
+   source .venv/bin/activate
+   export PYTHONPATH=$(pwd)
+   bash docker/launch_backend_service.sh
+   ```
+
+7. 安裝前端依賴：
+
+   ```bash
+   cd web
+   npm install
+   ```
+
+8. 啟動前端服務：
+
+   ```bash
+   npm run dev
+   ```
+
+   以下界面說明系統已成功啟動：_
+
+   ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
+   ```
+
+9. 開發完成後停止 RAGFlow 前端和後端服務：
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
+## 📚 技術文檔
+
+- [Quickstart](https://ragflow.io/docs/dev/)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
+- [References](https://ragflow.io/docs/dev/category/references)
+- [FAQs](https://ragflow.io/docs/dev/faq)
+
+## 📜 路線圖
+
+詳見 [RAGFlow Roadmap 2025](https://github.com/infiniflow/ragflow/issues/4214) 。
+
+## 🏄 開源社群
+
+- [Discord](https://discord.gg/zd4qPW6t)
+- [Twitter](https://twitter.com/infiniflowai)
+- [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
+
+## 🙌 貢獻指南
+
+RAGFlow 只有透過開源協作才能蓬勃發展。秉持這項精神,我們歡迎來自社區的各種貢獻。如果您有意參與其中,請查閱我們的 [貢獻者指南](https://ragflow.io/docs/dev/contributing) 。
+
+## 🤝 商務合作
+
+- [預約諮詢](https://aao615odquw.feishu.cn/share/base/form/shrcnjw7QleretCLqh1nuPo1xxh)
+
+## 👥 加入社區
+
+掃二維碼加入 RAGFlow 小助手，進 RAGFlow 交流群。
+
+<p align="center">
+  <img src="https://github.com/infiniflow/ragflow/assets/7248/bccf284f-46f2-4445-9809-8f1030fb7585" width=50% height=50%>
+</p>

README_zh.md

@@ -7,9 +7,11 @@
 <p align="center">
   <a href="./README.md">English</a> |
   <a href="./README_zh.md">简体中文</a> |
+  <a href="./README_tzh.md">繁体中文</a> |
   <a href="./README_ja.md">日本語</a> |
   <a href="./README_ko.md">한국어</a> |
-  <a href="./README_id.md">Bahasa Indonesia</a>
+  <a href="./README_id.md">Bahasa Indonesia</a> |
+  <a href="/README_pt_br.md">Português (Brasil)</a>
 </p>
 
 <p align="center">
@@ -20,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.14.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.14.0">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.19.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.19.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -30,12 +32,11 @@
     </a>
 </p>
 
-
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
-  <a href="https://github.com/infiniflow/ragflow/issues/162">Roadmap</a> |
+  <a href="https://github.com/infiniflow/ragflow/issues/4214">Roadmap</a> |
   <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/4XxujFgUN7">Discord</a> |
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
@@ -46,27 +47,29 @@
 ## 🎮 Demo 试用
 
 请登录网址 [https://demo.ragflow.io](https://demo.ragflow.io) 试用 demo。
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/infiniflow/ragflow/assets/7248/2f6baa3e-1092-4f11-866d-36f6a9d075e5" width="1200"/>
 <img src="https://github.com/user-attachments/assets/504bbbf1-c9f7-4d83-8cc5-e9cb63c26db6" width="1200"/>
 </div>
 
-
 ## 🔥 近期更新
 
-- 2024-11-22 完善了 Agent 中的变量定义和使用。
+- 2025-03-19 PDF和DOCX中的图支持用多模态大模型去解析得到描述.
+- 2025-02-28 结合互联网搜索（Tavily），对于任意大模型实现类似 Deep Research 的推理功能.
+- 2025-01-26 优化知识图谱的提取和应用，提供了多种配置选择。
+- 2024-12-18 升级了 DeepDoc 的文档布局分析模型。
 - 2024-11-01 对解析后的 chunk 加入关键词抽取和相关问题生成以提高召回的准确度。
-- 2024-09-13 增加知识库问答搜索模式。
 - 2024-08-22 支持用 RAG 技术实现从自然语言到 SQL 语句的转换。
-- 2024-08-02 支持 GraphRAG 启发于 [graphrag](https://github.com/microsoft/graphrag) 和思维导图。
 
 ## 🎉 关注项目
-⭐️点击右上角的 Star 关注RAGFlow，可以获取最新发布的实时通知 !🌟
+
+⭐️ 点击右上角的 Star 关注 RAGFlow，可以获取最新发布的实时通知 !🌟
+
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
 </div>
 
-
 ## 🌟 主要功能
 
 ### 🍭 **"Quality in, quality out"**
@@ -109,7 +112,10 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > 如果你并没有在本机安装 Docker（Windows、Mac，或者 Linux）, 可以参考文档 [Install Docker Engine](https://docs.docker.com/engine/install/) 自行安装。
+- [gVisor](https://gvisor.dev/docs/user_guide/install/): 仅在你打算使用 RAGFlow 的代码执行器（沙箱）功能时才需要安装。
+
+> [!TIP]
+> 如果你并没有在本机安装 Docker（Windows、Mac，或者 Linux）, 可以参考文档 [Install Docker Engine](https://docs.docker.com/engine/install/) 自行安装。
 
 ### 🚀 启动服务器
 
@@ -142,18 +148,34 @@
 
 3. 进入 **docker** 文件夹，利用提前编译好的 Docker 镜像启动服务器：
 
-   > 运行以下命令会自动下载 dev 版的 RAGFlow slim Docker 镜像（`dev-slim`），该镜像并不包含 embedding 模型以及一些 Python 库，因此镜像大小约 1GB。
+> [!CAUTION]
+> 请注意，目前官方提供的所有 Docker 镜像均基于 x86 架构构建，并不提供基于 ARM64 的 Docker 镜像。
+> 如果你的操作系统是 ARM64 架构，请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
+
+   > 运行以下命令会自动下载 RAGFlow slim Docker 镜像 `v0.19.0-slim`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.19.0-slim` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。比如，你可以通过设置 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.0` 来下载 RAGFlow 镜像的 `v0.19.0` 完整发行版。
 
    ```bash
    $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # docker compose -f docker-compose-gpu.yml up -d
    ```
 
-   > - 如果你想下载并运行特定版本的 RAGFlow slim Docker 镜像，请在 **docker/.env** 文件中找到 `RAGFLOW_IMAGE` 变量，将其改为对应版本。例如 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0-slim`，然后再运行上述命令。
-   > - 如果您想安装内置 embedding 模型和 Python 库的 dev 版本的 Docker 镜像，需要将 **docker/.env** 文件中的 `RAGFLOW_IMAGE` 变量修改为： `RAGFLOW_IMAGE=infiniflow/ragflow:dev`。
-   > - 如果您想安装内置 embedding 模型和 Python 库的指定版本的 RAGFlow Docker 镜像，需要将 **docker/.env** 文件中的 `RAGFLOW_IMAGE` 变量修改为： `RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0`。修改后，再运行上面的命令。
-   > **注意：** 安装内置 embedding 模型和 Python 库的指定版本的 RAGFlow Docker 镜像大小约 9 GB，可能需要更长时间下载，请耐心等待。
-   
+   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+   | ----------------- | --------------- | --------------------- | ------------------------ |
+   | v0.19.0           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.19.0-slim      | &approx;2       | ❌                    | Stable release           |
+   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
+   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+
+   > [!TIP]
+   > 如果你遇到 Docker 镜像拉不下来的问题，可以在 **docker/.env** 文件内根据变量 `RAGFLOW_IMAGE` 的注释提示选择华为云或者阿里云的相应镜像。
+   >
+   > - 华为云镜像名：`swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow`
+   > - 阿里云镜像名：`registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow`
+
 4. 服务器启动成功后再次确认服务器状态：
 
    ```bash
@@ -163,22 +185,20 @@
    _出现以下界面提示说明服务器启动成功：_
 
    ```bash
-        ____   ___    ______ ______ __               
+        ____   ___    ______ ______ __
        / __ \ /   |  / ____// ____// /____  _      __
       / /_/ // /| | / / __ / /_   / // __ \| | /| / /
-     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ / 
-    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/  
+     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
+    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
     * Running on all addresses (0.0.0.0)
-    * Running on http://127.0.0.1:9380
-    * Running on http://x.x.x.x:9380
-    INFO:werkzeug:Press CTRL+C to quit
    ```
-   > 如果您跳过这一步系统确认步骤就登录 RAGFlow，你的浏览器有可能会提示 `network anormal` 或 `网络异常`，因为 RAGFlow 可能并未完全启动成功。
+
+   > 如果您在没有看到上面的提示信息出来之前，就尝试登录 RAGFlow，你的浏览器有可能会提示 `network anormal` 或 `网络异常`。
 
 5. 在你的浏览器中输入你的服务器对应的 IP 地址并登录 RAGFlow。
    > 上面这个例子中，您只需输入 http://IP_OF_YOUR_MACHINE 即可：未改动过配置则无需输入端口（默认的 HTTP 服务端口 80）。
-6. 在 [service_conf.yaml](./docker/service_conf.yaml) 文件的 `user_default_llm` 栏配置 LLM factory，并在 `API_KEY` 栏填写和你选择的大模型相对应的 API key。
+6. 在 [service_conf.yaml.template](./docker/service_conf.yaml.template) 文件的 `user_default_llm` 栏配置 LLM factory，并在 `API_KEY` 栏填写和你选择的大模型相对应的 API key。
 
    > 详见 [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup)。
 
@@ -189,14 +209,14 @@
 系统配置涉及以下三份文件：
 
 - [.env](./docker/.env)：存放一些基本的系统环境变量，比如 `SVR_HTTP_PORT`、`MYSQL_PASSWORD`、`MINIO_PASSWORD` 等。
-- [service_conf.yaml](./docker/service_conf.yaml)：配置各类后台服务。
+- [service_conf.yaml.template](./docker/service_conf.yaml.template)：配置各类后台服务。
 - [docker-compose.yml](./docker/docker-compose.yml): 系统依赖该文件完成启动。
 
-请务必确保 [.env](./docker/.env) 文件中的变量设置与 [service_conf.yaml](./docker/service_conf.yaml) 文件中的配置保持一致！
+请务必确保 [.env](./docker/.env) 文件中的变量设置与 [service_conf.yaml.template](./docker/service_conf.yaml.template) 文件中的配置保持一致！
 
-如果不能访问镜像站点hub.docker.com或者模型站点huggingface.co，请按照[.env](./docker/.env)注释修改`RAGFLOW_IMAGE`和`HF_ENDPOINT`。
+如果不能访问镜像站点 hub.docker.com 或者模型站点 huggingface.co，请按照 [.env](./docker/.env) 注释修改 `RAGFLOW_IMAGE` 和 `HF_ENDPOINT`。
 
-> [./docker/README](./docker/README.md) 文件提供了环境变量设置和服务配置的详细信息。请**一定要**确保 [./docker/README](./docker/README.md) 文件当中列出来的环境变量的值与 [service_conf.yaml](./docker/service_conf.yaml) 文件当中的系统配置保持一致。
+> [./docker/README](./docker/README.md) 解释了 [service_conf.yaml.template](./docker/service_conf.yaml.template) 用到的环境变量设置和服务配置。
 
 如需更新默认的 HTTP 服务端口(80), 可以在 [docker-compose.yml](./docker/docker-compose.yml) 文件中将配置 `80:80` 改为 `<YOUR_SERVING_PORT>:80`。
 
@@ -215,29 +235,27 @@ RAGFlow 默认使用 Elasticsearch 存储文本和向量数据. 如果要切换
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+   Note: `-v` 将会删除 docker 容器的 volumes，已有的数据会被清空。
 
 2. 设置 **docker/.env** 目录中的 `DOC_ENGINE` 为 `infinity`.
 
 3. 启动容器:
 
    ```bash
-   $ docker compose -f docker/docker-compose.yml up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
 
-> [!WARNING] 
+> [!WARNING]
 > Infinity 目前官方并未正式支持在 Linux/arm64 架构下的机器上运行.
 
-
 ## 🔧 源码编译 Docker 镜像（不含 embedding 模型）
 
-本 Docker 镜像大小约 1 GB 左右并且依赖外部的大模型和 embedding 服务。
+本 Docker 镜像大小约 2 GB 左右并且依赖外部的大模型和 embedding 服务。
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+docker build --platform linux/amd64 --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 ```
 
 ## 🔧 源码编译 Docker 镜像（包含 embedding 模型）
@@ -247,85 +265,109 @@ docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub nltk
-python3 download_deps.py
-docker build -f Dockerfile -t infiniflow/ragflow:dev .
+docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 以源代码启动服务
 
-1. 安装 Poetry。如已经安装，可跳过本步骤：  
+1. 安装 uv。如已经安装，可跳过本步骤：
+
    ```bash
-   curl -sSL https://install.python-poetry.org | python3 -
+   pipx install uv pre-commit
+   export UV_INDEX=https://mirrors.aliyun.com/pypi/simple
    ```
 
-2. 下载源代码并安装 Python 依赖：  
+2. 下载源代码并安装 Python 依赖：
+
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   export POETRY_VIRTUALENVS_CREATE=true POETRY_VIRTUALENVS_IN_PROJECT=true
-   ~/.local/bin/poetry install --sync --no-root # install RAGFlow dependent python modules
+   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv run download_deps.py
+   pre-commit install
    ```
 
-3. 通过 Docker Compose 启动依赖的服务（MinIO, Elasticsearch, Redis, and MySQL）：  
+3. 通过 Docker Compose 启动依赖的服务（MinIO, Elasticsearch, Redis, and MySQL）：
+
    ```bash
    docker compose -f docker/docker-compose-base.yml up -d
    ```
 
-   在 `/etc/hosts` 中添加以下代码，将 **docker/service_conf.yaml** 文件中的所有 host 地址都解析为 `127.0.0.1`：  
-   ```
-   127.0.0.1       es01 infinity mysql minio redis
-   ```  
-   在文件 **docker/service_conf.yaml** 中，对照 **docker/.env** 的配置将 mysql 端口更新为 `5455`，es 端口更新为 `1200`。
+   在 `/etc/hosts` 中添加以下代码，目的是将 **conf/service_conf.yaml** 文件中的所有 host 地址都解析为 `127.0.0.1`：
+
+   ```
+   127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
+   ```
+4. 如果无法访问 HuggingFace，可以把环境变量 `HF_ENDPOINT` 设成相应的镜像站点：
 
-4. 如果无法访问 HuggingFace，可以把环境变量 `HF_ENDPOINT` 设成相应的镜像站点：  
- 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. 启动后端服务：  
+5. 如果你的操作系统没有 jemalloc，请按照如下方式安装：
+
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum install jemalloc
+   ```
+
+6. 启动后端服务：
+
    ```bash
    source .venv/bin/activate
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
 
-6. 安装前端依赖：  
+7. 安装前端依赖：
+
    ```bash
    cd web
-   npm install --force
-   ```  
-7. 配置前端，将 **.umirc.ts** 的 `proxy.target` 更新为 `http://127.0.0.1:9380`：  
-8. 启动前端服务：  
-   ```bash
-   npm run dev 
-   ```  
+   npm install
+   ```
 
-   _以下界面说明系统已经成功启动：_  
+8. 启动前端服务：
+
+   ```bash
+   npm run dev
+   ```
+
+   _以下界面说明系统已经成功启动：_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
+9. 开发完成后停止 RAGFlow 前端和后端服务：
+
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
+
+
 ## 📚 技术文档
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/guides)
+- [Configuration](https://ragflow.io/docs/dev/configurations)
+- [Release notes](https://ragflow.io/docs/dev/release_notes)
+- [User guides](https://ragflow.io/docs/dev/category/guides)
+- [Developer guides](https://ragflow.io/docs/dev/category/developers)
 - [References](https://ragflow.io/docs/dev/category/references)
-- [FAQ](https://ragflow.io/docs/dev/faq)
+- [FAQs](https://ragflow.io/docs/dev/faq)
 
 ## 📜 路线图
 
-详见 [RAGFlow Roadmap 2024](https://github.com/infiniflow/ragflow/issues/162) 。
+详见 [RAGFlow Roadmap 2025](https://github.com/infiniflow/ragflow/issues/4214) 。
 
 ## 🏄 开源社区
 
-- [Discord](https://discord.gg/4XxujFgUN7)
+- [Discord](https://discord.gg/zd4qPW6t)
 - [Twitter](https://twitter.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 贡献指南
 
-RAGFlow 只有通过开源协作才能蓬勃发展。秉持这一精神,我们欢迎来自社区的各种贡献。如果您有意参与其中,请查阅我们的 [贡献者指南](./CONTRIBUTING.md) 。
+RAGFlow 只有通过开源协作才能蓬勃发展。秉持这一精神,我们欢迎来自社区的各种贡献。如果您有意参与其中,请查阅我们的 [贡献者指南](https://ragflow.io/docs/dev/contributing) 。
 
 ## 🤝 商务合作
 
@@ -338,4 +380,3 @@ RAGFlow 只有通过开源协作才能蓬勃发展。秉持这一精神,我们
 <p align="center">
   <img src="https://github.com/infiniflow/ragflow/assets/7248/bccf284f-46f2-4445-9809-8f1030fb7585" width=50% height=50%>
 </p>
-

agent/README.md

@@ -10,7 +10,7 @@ It is used to compose a complex work flow or agent.
 And this graph is beyond the DAG that we can use circles to describe our agent or work flow.
 Under this folder, we propose a test tool ./test/client.py which can test the DSLs such as json files in folder ./test/dsl_examples.
 Please use this client at the same folder you start RAGFlow. If it's run by Docker, please go into the container before running the client.
-Otherwise, correct configurations in conf/service_conf.yaml is essential.
+Otherwise, correct configurations in service_conf.yaml is essential.
 
 ```bash
 PYTHONPATH=path/to/ragflow python graph/test/client.py -h

agent/README_zh.md

@@ -11,7 +11,7 @@
 在这个文件夹下，我们提出了一个测试工具 ./test/client.py，
 它可以测试像文件夹./test/dsl_examples下一样的DSL文件。
 请在启动 RAGFlow 的同一文件夹中使用此客户端。如果它是通过 Docker 运行的，请在运行客户端之前进入容器。
-否则，正确配置 conf/service_conf.yaml 文件是必不可少的。
+否则，正确配置 service_conf.yaml 文件是必不可少的。
 
 ```bash
 PYTHONPATH=path/to/ragflow python graph/test/client.py -h

agent/__init__.py

@@ -0,0 +1,18 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from beartype.claw import beartype_this_package
+beartype_this_package()

agent/canvas.py

@@ -15,13 +15,15 @@
 #
 import logging
 import json
-from abc import ABC
 from copy import deepcopy
 from functools import partial
+import pandas as pd
+
 from agent.component import component_class
 from agent.component.base import ComponentBase
 
-class Canvas(ABC):
+
+class Canvas:
     """
     dsl = {
         "components": {
@@ -82,7 +84,8 @@ class Canvas(ABC):
                         }
                     },
                     "downstream": [],
-                    "upstream": []
+                    "upstream": [],
+                    "parent_id": ""
                 }
             },
             "history": [],
@@ -133,7 +136,8 @@ class Canvas(ABC):
             "components": {}
         }
         for k in self.dsl.keys():
-            if k in ["components"]:continue
+            if k in ["components"]:
+                continue
             dsl[k] = deepcopy(self.dsl[k])
 
         for k, cpn in self.components.items():
@@ -156,12 +160,16 @@ class Canvas(ABC):
             self.components[k]["obj"].reset()
         self._embed_id = ""
 
-    def get_compnent_name(self, cid):
+    def get_component_name(self, cid):
         for n in self.dsl["graph"]["nodes"]:
-            if cid == n["id"]: return n["data"]["name"]
+            if cid == n["id"]:
+                return n["data"]["name"]
         return ""
 
-    def run(self, **kwargs):
+    def run(self, running_hint_text = "is running...🕞", **kwargs):
+        if not running_hint_text or not isinstance(running_hint_text, str):
+            running_hint_text = "is running...🕞"
+
         if self.answer:
             cpn_id = self.answer[0]
             self.answer.pop(0)
@@ -173,7 +181,8 @@ class Canvas(ABC):
             if kwargs.get("stream"):
                 for an in ans():
                     yield an
-            else: yield ans
+            else:
+                yield ans
             return
 
         if not self.path:
@@ -181,6 +190,7 @@ class Canvas(ABC):
             self.path.append(["begin"])
 
         self.path.append([])
+
         ran = -1
         waiting = []
         without_dependent_checking = []
@@ -188,7 +198,8 @@ class Canvas(ABC):
         def prepare2run(cpns):
             nonlocal ran, ans
             for c in cpns:
-                if self.path[-1] and c == self.path[-1][-1]: continue
+                if self.path[-1] and c == self.path[-1][-1]:
+                    continue
                 cpn = self.components[c]["obj"]
                 if cpn.component_name == "Answer":
                     self.answer.append(c)
@@ -197,49 +208,77 @@ class Canvas(ABC):
                     if c not in without_dependent_checking:
                         cpids = cpn.get_dependent_components()
                         if any([cc not in self.path[-1] for cc in cpids]):
-                            if c not in waiting: waiting.append(c)
+                            if c not in waiting:
+                                waiting.append(c)
                             continue
-                    yield "*'{}'* is running...🕞".format(self.get_compnent_name(c))
-                    ans = cpn.run(self.history, **kwargs)
+                    yield "*'{}'* {}".format(self.get_component_name(c), running_hint_text)
+
+                    if cpn.component_name.lower() == "iteration":
+                        st_cpn = cpn.get_start()
+                        assert st_cpn, "Start component not found for Iteration."
+                        if not st_cpn["obj"].end():
+                            cpn = st_cpn["obj"]
+                            c = cpn._id
+
+                    try:
+                        ans = cpn.run(self.history, **kwargs)
+                    except Exception as e:
+                        logging.exception(f"Canvas.run got exception: {e}")
+                        self.path[-1].append(c)
+                        ran += 1
+                        raise e
                     self.path[-1].append(c)
+
             ran += 1
 
-        for m in prepare2run(self.components[self.path[-2][-1]]["downstream"]):
+        downstream = self.components[self.path[-2][-1]]["downstream"]
+        if not downstream and self.components[self.path[-2][-1]].get("parent_id"):
+            cid = self.path[-2][-1]
+            pid = self.components[cid]["parent_id"]
+            o, _ = self.components[cid]["obj"].output(allow_partial=False)
+            oo, _ = self.components[pid]["obj"].output(allow_partial=False)
+            self.components[pid]["obj"].set_output(pd.concat([oo, o], ignore_index=True).dropna())
+            downstream = [pid]
+
+        for m in prepare2run(downstream):
             yield {"content": m, "running_status": True}
 
         while 0 <= ran < len(self.path[-1]):
             logging.debug(f"Canvas.run: {ran} {self.path}")
             cpn_id = self.path[-1][ran]
             cpn = self.get_component(cpn_id)
-            if not cpn["downstream"]: break
+            if not any([cpn["downstream"], cpn.get("parent_id"), waiting]):
+                break
 
             loop = self._find_loop()
-            if loop: raise OverflowError(f"Too much loops: {loop}")
+            if loop:
+                raise OverflowError(f"Too much loops: {loop}")
 
+            downstream = []
             if cpn["obj"].component_name.lower() in ["switch", "categorize", "relevant"]:
                 switch_out = cpn["obj"].output()[1].iloc[0, 0]
                 assert switch_out in self.components, \
                     "{}'s output: {} not valid.".format(cpn_id, switch_out)
-                try:
-                    for m in prepare2run([switch_out]):
-                        yield {"content": m, "running_status": True}
-                except Exception as e:
-                    yield {"content": "*Exception*: {}".format(e), "running_status": True}
-                    logging.exception("Canvas.run got exception")
-                continue
+                downstream = [switch_out]
+            else:
+                downstream = cpn["downstream"]
 
-            try:
-                for m in prepare2run(cpn["downstream"]):
-                    yield {"content": m, "running_status": True}
-            except Exception as e:
-                yield {"content": "*Exception*: {}".format(e), "running_status": True}
-                logging.exception("Canvas.run got exception")
+            if not downstream and cpn.get("parent_id"):
+                pid = cpn["parent_id"]
+                _, o = cpn["obj"].output(allow_partial=False)
+                _, oo = self.components[pid]["obj"].output(allow_partial=False)
+                self.components[pid]["obj"].set_output(pd.concat([oo.dropna(axis=1), o.dropna(axis=1)], ignore_index=True).dropna())
+                downstream = [pid]
+
+            for m in prepare2run(downstream):
+                yield {"content": m, "running_status": True}
 
             if ran >= len(self.path[-1]) and waiting:
                 without_dependent_checking = waiting
                 waiting = []
                 for m in prepare2run(without_dependent_checking):
                     yield {"content": m, "running_status": True}
+                without_dependent_checking = []
                 ran -= 1
 
         if self.answer:
@@ -283,19 +322,22 @@ class Canvas(ABC):
 
     def _find_loop(self, max_loops=6):
         path = self.path[-1][::-1]
-        if len(path) < 2: return False
+        if len(path) < 2:
+            return False
 
         for i in range(len(path)):
-            if path[i].lower().find("answer") >= 0:
+            if path[i].lower().find("answer") == 0 or path[i].lower().find("iterationitem") == 0:
                 path = path[:i]
                 break
 
-        if len(path) < 2: return False
+        if len(path) < 2:
+            return False
 
-        for l in range(2, len(path) // 2):
-            pat = ",".join(path[0:l])
+        for loc in range(2, len(path) // 2):
+            pat = ",".join(path[0:loc])
             path_str = ",".join(path)
-            if len(pat) >= len(path_str): return False
+            if len(pat) >= len(path_str):
+                return False
             loop = max_loops
             while path_str.find(pat) == 0 and loop >= 0:
                 loop -= 1
@@ -303,10 +345,26 @@ class Canvas(ABC):
                     return False
                 path_str = path_str[len(pat)+1:]
             if loop < 0:
-                pat = " => ".join([p.split(":")[0] for p in path[0:l]])
+                pat = " => ".join([p.split(":")[0] for p in path[0:loc]])
                 return pat + " => " + pat
 
         return False
 
     def get_prologue(self):
         return self.components["begin"]["obj"]._param.prologue
+
+    def set_global_param(self, **kwargs):
+        for k, v in kwargs.items():
+            for q in self.components["begin"]["obj"]._param.query:
+                if k != q["key"]:
+                    continue
+                q["value"] = v
+
+    def get_preset_param(self):
+        return self.components["begin"]["obj"]._param.query
+
+    def get_component_input_elements(self, cpnnm):
+        return self.components[cpnnm]["obj"].get_input_elements()
+    
+    def set_component_infor(self, cpn_id, infor):
+        self.components[cpn_id]["obj"].set_infor(infor)

agent/component/__init__.py

@@ -1,3 +1,19 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
 import importlib
 from .begin import Begin, BeginParam
 from .generate import Generate, GenerateParam
@@ -31,9 +47,90 @@ from .akshare import AkShare, AkShareParam
 from .crawler import Crawler, CrawlerParam
 from .invoke import Invoke, InvokeParam
 from .template import Template, TemplateParam
+from .email import Email, EmailParam
+from .iteration import Iteration, IterationParam
+from .iterationitem import IterationItem, IterationItemParam
+from .code import Code, CodeParam
 
 
 def component_class(class_name):
     m = importlib.import_module("agent.component")
     c = getattr(m, class_name)
     return c
+
+
+__all__ = [
+    "Begin",
+    "BeginParam",
+    "Generate",
+    "GenerateParam",
+    "Retrieval",
+    "RetrievalParam",
+    "Answer",
+    "AnswerParam",
+    "Categorize",
+    "CategorizeParam",
+    "Switch",
+    "SwitchParam",
+    "Relevant",
+    "RelevantParam",
+    "Message",
+    "MessageParam",
+    "RewriteQuestion",
+    "RewriteQuestionParam",
+    "KeywordExtract",
+    "KeywordExtractParam",
+    "Concentrator",
+    "ConcentratorParam",
+    "Baidu",
+    "BaiduParam",
+    "DuckDuckGo",
+    "DuckDuckGoParam",
+    "Wikipedia",
+    "WikipediaParam",
+    "PubMed",
+    "PubMedParam",
+    "ArXiv",
+    "ArXivParam",
+    "Google",
+    "GoogleParam",
+    "Bing",
+    "BingParam",
+    "GoogleScholar",
+    "GoogleScholarParam",
+    "DeepL",
+    "DeepLParam",
+    "GitHub",
+    "GitHubParam",
+    "BaiduFanyi",
+    "BaiduFanyiParam",
+    "QWeather",
+    "QWeatherParam",
+    "ExeSQL",
+    "ExeSQLParam",
+    "YahooFinance",
+    "YahooFinanceParam",
+    "WenCai",
+    "WenCaiParam",
+    "Jin10",
+    "Jin10Param",
+    "TuShare",
+    "TuShareParam",
+    "AkShare",
+    "AkShareParam",
+    "Crawler",
+    "CrawlerParam",
+    "Invoke",
+    "InvokeParam",
+    "Iteration",
+    "IterationParam",
+    "IterationItem",
+    "IterationItemParam",
+    "Template",
+    "TemplateParam",
+    "Email",
+    "EmailParam",
+    "Code",
+    "CodeParam",
+    "component_class"
+]

agent/component/akshare.py

@@ -1,56 +1,56 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-from abc import ABC
-import pandas as pd
-import akshare as ak
-from agent.component.base import ComponentBase, ComponentParamBase
-
-
-class AkShareParam(ComponentParamBase):
-    """
-    Define the AkShare component parameters.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.top_n = 10
-
-    def check(self):
-        self.check_positive_integer(self.top_n, "Top N")
-
-
-class AkShare(ComponentBase, ABC):
-    component_name = "AkShare"
-
-    def _run(self, history, **kwargs):
-        ans = self.get_input()
-        ans = ",".join(ans["content"]) if "content" in ans else ""
-        if not ans:
-            return AkShare.be_output("")
-
-        try:
-            ak_res = []
-            stock_news_em_df = ak.stock_news_em(symbol=ans)
-            stock_news_em_df = stock_news_em_df.head(self._param.top_n)
-            ak_res = [{"content": '<a href="' + i["新闻链接"] + '">' + i["新闻标题"] + '</a>\n 新闻内容: ' + i[
-                "新闻内容"] + " \n发布时间:" + i["发布时间"] + " \n文章来源: " + i["文章来源"]} for index, i in stock_news_em_df.iterrows()]
-        except Exception as e:
-            return AkShare.be_output("**ERROR**: " + str(e))
-
-        if not ak_res:
-            return AkShare.be_output("")
-
-        return pd.DataFrame(ak_res)
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from abc import ABC
+import pandas as pd
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class AkShareParam(ComponentParamBase):
+    """
+    Define the AkShare component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.top_n = 10
+
+    def check(self):
+        self.check_positive_integer(self.top_n, "Top N")
+
+
+class AkShare(ComponentBase, ABC):
+    component_name = "AkShare"
+
+    def _run(self, history, **kwargs):
+        import akshare as ak
+        ans = self.get_input()
+        ans = ",".join(ans["content"]) if "content" in ans else ""
+        if not ans:
+            return AkShare.be_output("")
+
+        try:
+            ak_res = []
+            stock_news_em_df = ak.stock_news_em(symbol=ans)
+            stock_news_em_df = stock_news_em_df.head(self._param.top_n)
+            ak_res = [{"content": '<a href="' + i["新闻链接"] + '">' + i["新闻标题"] + '</a>\n 新闻内容: ' + i[
+                "新闻内容"] + " \n发布时间:" + i["发布时间"] + " \n文章来源: " + i["文章来源"]} for index, i in stock_news_em_df.iterrows()]
+        except Exception as e:
+            return AkShare.be_output("**ERROR**: " + str(e))
+
+        if not ak_res:
+            return AkShare.be_output("")
+
+        return pd.DataFrame(ak_res)

agent/component/answer.py

@@ -16,6 +16,7 @@
 import random
 from abc import ABC
 from functools import partial
+from typing import Tuple, Union
 
 import pandas as pd
 
@@ -76,4 +77,13 @@ class Answer(ComponentBase, ABC):
     def set_exception(self, e):
         self.exception = e
 
+    def output(self, allow_partial=True) -> Tuple[str, Union[pd.DataFrame, partial]]:
+        if allow_partial:
+            return super.output()
+
+        for r, c in self._canvas.history[::-1]:
+            if r == "user":
+                return self._param.output_var_name, pd.DataFrame([{"content": c}])
+
+        self._param.output_var_name, pd.DataFrame([])
 

agent/component/baidu.py

@@ -17,6 +17,7 @@ import logging
 from abc import ABC
 import pandas as pd
 import requests
+from bs4 import BeautifulSoup
 import re
 from agent.component.base import ComponentBase, ComponentParamBase
 
@@ -46,15 +47,26 @@ class Baidu(ComponentBase, ABC):
         try:
             url = 'https://www.baidu.com/s?wd=' + ans + '&rn=' + str(self._param.top_n)
             headers = {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'}
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+                'Connection': 'keep-alive',
+            }
             response = requests.get(url=url, headers=headers)
-
-            url_res = re.findall(r"'url': \\\"(.*?)\\\"}", response.text)
-            title_res = re.findall(r"'title': \\\"(.*?)\\\",\\n", response.text)
-            body_res = re.findall(r"\"contentText\":\"(.*?)\"", response.text)
-            baidu_res = [{"content": re.sub('<em>|</em>', '', '<a href="' + url + '">' + title + '</a>    ' + body)} for
-                         url, title, body in zip(url_res, title_res, body_res)]
-            del body_res, url_res, title_res
+            # check if request success
+            if response.status_code == 200:
+                soup = BeautifulSoup(response.text, 'html.parser')
+                url_res = []
+                title_res = []
+                body_res = []
+                for item in soup.select('.result.c-container'):
+                    # extract title
+                    title_res.append(item.select_one('h3 a').get_text(strip=True))
+                    url_res.append(item.select_one('h3 a')['href'])
+                    body_res.append(item.select_one('.c-abstract').get_text(strip=True) if item.select_one('.c-abstract') else '')
+                baidu_res = [{"content": re.sub('<em>|</em>', '', '<a href="' + url + '">' + title + '</a>    ' + body)} for
+                             url, title, body in zip(url_res, title_res, body_res)]
+                del body_res, url_res, title_res
         except Exception as e:
             return Baidu.be_output("**ERROR**: " + str(e))
 

agent/component/baidufanyi.py

@@ -39,9 +39,6 @@ class BaiduFanyiParam(ComponentParamBase):
         self.check_empty(self.appid, "BaiduFanyi APPID")
         self.check_empty(self.secret_key, "BaiduFanyi Secret Key")
         self.check_valid_value(self.trans_type, "Translate type", ['translate', 'fieldtranslate'])
-        self.check_valid_value(self.trans_type, "Translate domain",
-                               ['it', 'finance', 'machinery', 'senimed', 'novel', 'academic', 'aerospace', 'wiki',
-                                'news', 'law', 'contract'])
         self.check_valid_value(self.source_lang, "Source language",
                                ['auto', 'zh', 'en', 'yue', 'wyw', 'jp', 'kor', 'fra', 'spa', 'th', 'ara', 'ru', 'pt',
                                 'de', 'it', 'el', 'nl', 'pl', 'bul', 'est', 'dan', 'fin', 'cs', 'rom', 'slo', 'swe',
@@ -96,3 +93,4 @@ class BaiduFanyi(ComponentBase, ABC):
 
         except Exception as e:
             BaiduFanyi.be_output("**Error**:" + str(e))
+    

agent/component/base.py

@@ -19,7 +19,7 @@ import json
 import os
 import logging
 from functools import partial
-from typing import Tuple, Union
+from typing import Any, Tuple, Union
 
 import pandas as pd
 
@@ -34,9 +34,11 @@ _IS_RAW_CONF = "_is_raw_conf"
 class ComponentParamBase(ABC):
     def __init__(self):
         self.output_var_name = "output"
+        self.infor_var_name = "infor"
         self.message_history_window_size = 22
         self.query = []
         self.inputs = []
+        self.debug_inputs = []
 
     def set_name(self, name: str):
         self._name = name
@@ -383,6 +385,11 @@ class ComponentBase(ABC):
             "params": {}
         }
         """
+        out = getattr(self._param, self._param.output_var_name)
+        if isinstance(out, pd.DataFrame) and "chunks" in out:
+            del out["chunks"]
+            setattr(self._param, self._param.output_var_name, out)
+
         return """{{
             "component_name": "{}",
             "params": {},
@@ -395,6 +402,8 @@ class ComponentBase(ABC):
         )
 
     def __init__(self, canvas, id, param: ComponentParamBase):
+        from agent.canvas import Canvas  # Local import to avoid cyclic dependency
+        assert isinstance(canvas, Canvas), "canvas must be an instance of Canvas"
         self._canvas = canvas
         self._id = id
         self._param = param
@@ -410,6 +419,7 @@ class ComponentBase(ABC):
     def run(self, history, **kwargs):
         logging.debug("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False),
                                                               json.dumps(kwargs, ensure_ascii=False)))
+        self._param.debug_inputs = []
         try:
             res = self._run(history, **kwargs)
             self.set_output(res)
@@ -424,68 +434,106 @@ class ComponentBase(ABC):
 
     def output(self, allow_partial=True) -> Tuple[str, Union[pd.DataFrame, partial]]:
         o = getattr(self._param, self._param.output_var_name)
-        if not isinstance(o, partial) and not isinstance(o, pd.DataFrame):
-            if not isinstance(o, list): o = [o]
-            o = pd.DataFrame(o)
+        if not isinstance(o, partial):
+            if not isinstance(o, pd.DataFrame):
+                if isinstance(o, list):
+                    return self._param.output_var_name, pd.DataFrame(o).dropna()
+                if o is None:
+                    return self._param.output_var_name, pd.DataFrame()
+                return self._param.output_var_name, pd.DataFrame([{"content": str(o)}])
+            return self._param.output_var_name, o
 
         if allow_partial or not isinstance(o, partial):
             if not isinstance(o, partial) and not isinstance(o, pd.DataFrame):
-                return pd.DataFrame(o if isinstance(o, list) else [o])
+                return pd.DataFrame(o if isinstance(o, list) else [o]).dropna()
             return self._param.output_var_name, o
 
         outs = None
         for oo in o():
             if not isinstance(oo, pd.DataFrame):
-                outs = pd.DataFrame(oo if isinstance(oo, list) else [oo])
-            else: outs = oo
+                outs = pd.DataFrame(oo if isinstance(oo, list) else [oo]).dropna()
+            else:
+                outs = oo.dropna()
         return self._param.output_var_name, outs
 
     def reset(self):
         setattr(self._param, self._param.output_var_name, None)
         self._param.inputs = []
 
-    def set_output(self, v: partial | pd.DataFrame):
+    def set_output(self, v):
         setattr(self._param, self._param.output_var_name, v)
 
+    def set_infor(self, v):
+        setattr(self._param, self._param.infor_var_name, v)
+        
+    def _fetch_outputs_from(self, sources: list[dict[str, Any]]) -> list[pd.DataFrame]:
+        outs = []
+        for q in sources:
+            if q.get("component_id"):
+                if "@" in q["component_id"] and q["component_id"].split("@")[0].lower().find("begin") >= 0:
+                    cpn_id, key = q["component_id"].split("@")
+                    for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
+                        if p["key"] == key:
+                            outs.append(pd.DataFrame([{"content": p.get("value", "")}]))
+                            break
+                    else:
+                        assert False, f"Can't find parameter '{key}' for {cpn_id}"
+                    continue
+
+                if q["component_id"].lower().find("answer") == 0:
+                    txt = []
+                    for r, c in self._canvas.history[::-1][:self._param.message_history_window_size][::-1]:
+                        txt.append(f"{r.upper()}:{c}")
+                    txt = "\n".join(txt)
+                    outs.append(pd.DataFrame([{"content": txt}]))
+                    continue
+
+                outs.append(self._canvas.get_component(q["component_id"])["obj"].output(allow_partial=False)[1])
+            elif q.get("value"):
+                outs.append(pd.DataFrame([{"content": q["value"]}]))
+        return outs
     def get_input(self):
+        if self._param.debug_inputs:
+            return pd.DataFrame([{"content": v["value"]} for v in self._param.debug_inputs if v.get("value")])
+
         reversed_cpnts = []
         if len(self._canvas.path) > 1:
             reversed_cpnts.extend(self._canvas.path[-2])
         reversed_cpnts.extend(self._canvas.path[-1])
+        up_cpns = self.get_upstream()
+        reversed_up_cpnts = [cpn for cpn in reversed_cpnts if cpn in up_cpns]
 
         if self._param.query:
             self._param.inputs = []
-            outs = []
-            for q in self._param.query:
-                if q["component_id"]:
-                    if q["component_id"].split("@")[0].lower().find("begin") >= 0:
-                        cpn_id, key = q["component_id"].split("@")
-                        for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
-                            if p["key"] == key:
-                                outs.append(pd.DataFrame([{"content": p.get("value", "")}]))
-                                self._param.inputs.append({"component_id": q["component_id"],
-                                                           "content": p.get("value", "")})
-                                break
-                        else:
-                            assert False, f"Can't find parameter '{key}' for {cpn_id}"
-                        continue
+            outs = self._fetch_outputs_from(self._param.query)
+
+            for out in outs:
+                records = out.to_dict("records")
+                content: str
+
+                if len(records) > 1:
+                    content = "\n".join(
+                        [str(d["content"]) for d in records]
+                    )
+                else:
+                    content = records[0]["content"]
+
+                self._param.inputs.append({
+                    "component_id": records[0].get("component_id"),
+                    "content": content
+                })
 
-                    outs.append(self._canvas.get_component(q["component_id"])["obj"].output(allow_partial=False)[1])
-                    self._param.inputs.append({"component_id": q["component_id"],
-                                               "content": "\n".join(
-                                                   [str(d["content"]) for d in outs[-1].to_dict('records')])})
-                elif q["value"]:
-                    self._param.inputs.append({"component_id": None, "content": q["value"]})
-                    outs.append(pd.DataFrame([{"content": q["value"]}]))
             if outs:
                 df = pd.concat(outs, ignore_index=True)
-                if "content" in df: df = df.drop_duplicates(subset=['content']).reset_index(drop=True)
+                if "content" in df:
+                    df = df.drop_duplicates(subset=['content']).reset_index(drop=True)
                 return df
 
         upstream_outs = []
 
-        for u in reversed_cpnts[::-1]:
-            if self.get_component_name(u) in ["switch", "concentrator"]: continue
+        for u in reversed_up_cpnts[::-1]:
+            if self.get_component_name(u) in ["switch", "concentrator"]:
+                continue
             if self.component_name.lower() == "generate" and self.get_component_name(u) == "retrieval":
                 o = self._canvas.get_component(u)["obj"].output(allow_partial=False)[1]
                 if o is not None:
@@ -522,14 +570,33 @@ class ComponentBase(ABC):
 
         return df
 
+    def get_input_elements(self):
+        assert self._param.query, "Please verify the input parameters first."
+        eles = []
+        for q in self._param.query:
+            if q.get("component_id"):
+                cpn_id = q["component_id"]
+                if cpn_id.split("@")[0].lower().find("begin") >= 0:
+                    cpn_id, key = cpn_id.split("@")
+                    eles.extend(self._canvas.get_component(cpn_id)["obj"]._param.query)
+                    continue
+
+                eles.append({"name": self._canvas.get_component_name(cpn_id), "key": cpn_id})
+            else:
+                eles.append({"key": q["value"], "name": q["value"], "value": q["value"]})
+        return eles
+
     def get_stream_input(self):
         reversed_cpnts = []
         if len(self._canvas.path) > 1:
             reversed_cpnts.extend(self._canvas.path[-2])
         reversed_cpnts.extend(self._canvas.path[-1])
+        up_cpns = self.get_upstream()
+        reversed_up_cpnts = [cpn for cpn in reversed_cpnts if cpn in up_cpns]
 
-        for u in reversed_cpnts[::-1]:
-            if self.get_component_name(u) in ["switch", "answer"]: continue
+        for u in reversed_up_cpnts[::-1]:
+            if self.get_component_name(u) in ["switch", "answer"]:
+                continue
             return self._canvas.get_component(u)["obj"].output()[1]
 
     @staticmethod
@@ -538,3 +605,14 @@ class ComponentBase(ABC):
 
     def get_component_name(self, cpn_id):
         return self._canvas.get_component(cpn_id)["obj"].component_name.lower()
+
+    def debug(self, **kwargs):
+        return self._run([], **kwargs)
+
+    def get_parent(self):
+        pid = self._canvas.get_component(self._id)["parent_id"]
+        return self._canvas.get_component(pid)["obj"]
+
+    def get_upstream(self):
+        cpn_nms = self._canvas.get_component(self._id)['upstream']
+        return cpn_nms

agent/component/begin.py

@@ -26,6 +26,7 @@ class BeginParam(ComponentParamBase):
     def __init__(self):
         super().__init__()
         self.prologue = "Hi! I'm your smart assistant. What can I do for you?"
+        self.query = []
 
     def check(self):
         return True
@@ -42,7 +43,7 @@ class Begin(ComponentBase):
     def stream_output(self):
         res = {"content": self._param.prologue}
         yield res
-        self.set_output(res)
+        self.set_output(self.be_output(res))
 
 
 

agent/component/categorize.py

@@ -34,36 +34,46 @@ class CategorizeParam(GenerateParam):
         super().check()
         self.check_empty(self.category_description, "[Categorize] Category examples")
         for k, v in self.category_description.items():
-            if not k: raise ValueError("[Categorize] Category name can not be empty!")
-            if not v.get("to"): raise ValueError(f"[Categorize] 'To' of category {k} can not be empty!")
+            if not k:
+                raise ValueError("[Categorize] Category name can not be empty!")
+            if not v.get("to"):
+                raise ValueError(f"[Categorize] 'To' of category {k} can not be empty!")
 
-    def get_prompt(self):
+    def get_prompt(self, chat_hist):
         cate_lines = []
         for c, desc in self.category_description.items():
-            for l in desc.get("examples", "").split("\n"):
-                if not l: continue
-                cate_lines.append("Question: {}\tCategory: {}".format(l, c))
+            for line in desc.get("examples", "").split("\n"):
+                if not line:
+                    continue
+                cate_lines.append("USER: {}\nCategory: {}".format(line, c))
         descriptions = []
         for c, desc in self.category_description.items():
             if desc.get("description"):
                 descriptions.append(
-                    "--------------------\nCategory: {}\nDescription: {}\n".format(c, desc["description"]))
+                    "\nCategory: {}\nDescription: {}".format(c, desc["description"]))
 
         self.prompt = """
-        You're a text classifier. You need to categorize the user’s questions into {} categories, 
-        namely: {}
-        Here's description of each category:
-        {}
+Role: You're a text classifier. 
+Task: You need to categorize the user’s questions into {} categories, namely: {}
 
-        You could learn from the following examples:
-        {}
-        You could learn from the above examples.
-        Just mention the category names, no need for any additional words.
+Here's description of each category:
+{}
+
+You could learn from the following examples:
+{}
+You could learn from the above examples.
+
+Requirements:
+- Just mention the category names, no need for any additional words.
+
+---- Real Data ----
+USER: {}\n
         """.format(
             len(self.category_description.keys()),
             "/".join(list(self.category_description.keys())),
             "\n".join(descriptions),
-            "- ".join(cate_lines)
+            "\n\n- ".join(cate_lines),
+            chat_hist
         )
         return self.prompt
 
@@ -73,15 +83,28 @@ class Categorize(Generate, ABC):
 
     def _run(self, history, **kwargs):
         input = self.get_input()
-        input = "Question: " + (list(input["content"])[-1] if "content" in input else "") + "\tCategory: "
+        input = " - ".join(input["content"]) if "content" in input else ""
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
-        ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": input}],
+        self._canvas.set_component_infor(self._id, {"prompt":self._param.get_prompt(input),"messages":  [{"role": "user", "content": "\nCategory: "}],"conf": self._param.gen_conf()})
+
+        ans = chat_mdl.chat(self._param.get_prompt(input), [{"role": "user", "content": "\nCategory: "}],
                             self._param.gen_conf())
-        logging.debug(f"input: {input}, answer: {str(ans)}")
+        logging.debug(f"input: {input}, answer: {str(ans)}")    
+        # Count the number of times each category appears in the answer.
+        category_counts = {}
         for c in self._param.category_description.keys():
-            if ans.lower().find(c.lower()) >= 0:
-                return Categorize.be_output(self._param.category_description[c]["to"])
+            count = ans.lower().count(c.lower())
+            category_counts[c] = count
+            
+        # If a category is found, return the category with the highest count.
+        if any(category_counts.values()):
+            max_category = max(category_counts.items(), key=lambda x: x[1])
+            return Categorize.be_output(self._param.category_description[max_category[0]]["to"])
 
         return Categorize.be_output(list(self._param.category_description.items())[-1][1]["to"])
 
+    def debug(self, **kwargs):
+        df = self._run([], **kwargs)
+        cpn_id = df.iloc[0, 0]
+        return Categorize.be_output(self._canvas.get_component_name(cpn_id))
 

agent/component/code.py

@@ -0,0 +1,138 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import base64
+from abc import ABC
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+from agent.component.base import ComponentBase, ComponentParamBase
+from api import settings
+
+
+class Language(str, Enum):
+    PYTHON = "python"
+    NODEJS = "nodejs"
+
+
+class CodeExecutionRequest(BaseModel):
+    code_b64: str = Field(..., description="Base64 encoded code string")
+    language: Language = Field(default=Language.PYTHON, description="Programming language")
+    arguments: Optional[dict] = Field(default={}, description="Arguments")
+
+    @field_validator("code_b64")
+    @classmethod
+    def validate_base64(cls, v: str) -> str:
+        try:
+            base64.b64decode(v, validate=True)
+            return v
+        except Exception as e:
+            raise ValueError(f"Invalid base64 encoding: {str(e)}")
+
+    @field_validator("language", mode="before")
+    @classmethod
+    def normalize_language(cls, v) -> str:
+        if isinstance(v, str):
+            low = v.lower()
+            if low in ("python", "python3"):
+                return "python"
+            elif low in ("javascript", "nodejs"):
+                return "nodejs"
+        raise ValueError(f"Unsupported language: {v}")
+
+
+class CodeParam(ComponentParamBase):
+    """
+    Define the code sandbox component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.lang = "python"
+        self.script = ""
+        self.arguments = []
+        self.address = f"http://{settings.SANDBOX_HOST}:9385/run"
+        self.enable_network = True
+
+    def check(self):
+        self.check_valid_value(self.lang, "Support languages", ["python", "python3", "nodejs", "javascript"])
+        self.check_defined_type(self.enable_network, "Enable network", ["bool"])
+
+
+class Code(ComponentBase, ABC):
+    component_name = "Code"
+
+    def _run(self, history, **kwargs):
+        arguments = {}
+        for input in self._param.arguments:
+            if "@" in input["component_id"]:
+                component_id = input["component_id"].split("@")[0]
+                refered_component_key = input["component_id"].split("@")[1]
+                refered_component = self._canvas.get_component(component_id)["obj"]
+
+                for param in refered_component._param.query:
+                    if param["key"] == refered_component_key:
+                        if "value" in param:
+                            arguments[input["name"]] = param["value"]
+            else:
+                cpn = self._canvas.get_component(input["component_id"])["obj"]
+                if cpn.component_name.lower() == "answer":
+                    arguments[input["name"]] = self._canvas.get_history(1)[0]["content"]
+                    continue
+                _, out = cpn.output(allow_partial=False)
+                if not out.empty:
+                    arguments[input["name"]] = "\n".join(out["content"])
+
+        return self._execute_code(
+            language=self._param.lang,
+            code=self._param.script,
+            arguments=arguments,
+            address=self._param.address,
+            enable_network=self._param.enable_network,
+        )
+
+    def _execute_code(self, language: str, code: str, arguments: dict, address: str, enable_network: bool):
+        import requests
+
+        try:
+            code_b64 = self._encode_code(code)
+            code_req = CodeExecutionRequest(code_b64=code_b64, language=language, arguments=arguments).model_dump()
+        except Exception as e:
+            return Code.be_output("**Error**: construct code request error: " + str(e))
+
+        try:
+            resp = requests.post(url=address, json=code_req, timeout=10)
+            body = resp.json()
+            if body:
+                stdout = body.get("stdout")
+                stderr = body.get("stderr")
+                return Code.be_output(stdout or stderr)
+            else:
+                return Code.be_output("**Error**: There is no response from sanbox")
+
+        except Exception as e:
+            return Code.be_output("**Error**: Internal error in sanbox: " + str(e))
+
+    def _encode_code(self, code: str) -> str:
+        return base64.b64encode(code.encode("utf-8")).decode("utf-8")
+
+    def get_input_elements(self):
+        elements = []
+        for input in self._param.arguments:
+            cpn_id = input["component_id"]
+            elements.append({"key": cpn_id, "name": input["name"]})
+        return elements

agent/component/concentrator.py

@@ -1,36 +1,36 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-from abc import ABC
-from agent.component.base import ComponentBase, ComponentParamBase
-
-
-class ConcentratorParam(ComponentParamBase):
-    """
-    Define the Concentrator component parameters.
-    """
-
-    def __init__(self):
-        super().__init__()
-
-    def check(self):
-        return True
-
-
-class Concentrator(ComponentBase, ABC):
-    component_name = "Concentrator"
-
-    def _run(self, history, **kwargs):
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from abc import ABC
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class ConcentratorParam(ComponentParamBase):
+    """
+    Define the Concentrator component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    def check(self):
+        return True
+
+
+class Concentrator(ComponentBase, ABC):
+    component_name = "Concentrator"
+
+    def _run(self, history, **kwargs):
         return Concentrator.be_output("")

agent/component/crawler.py

@@ -17,6 +17,7 @@ from abc import ABC
 import asyncio
 from crawl4ai import AsyncWebCrawler
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.web_utils import is_valid_url
 
 
 class CrawlerParam(ComponentParamBase):
@@ -39,8 +40,8 @@ class Crawler(ComponentBase, ABC):
     def _run(self, history, **kwargs):
         ans = self.get_input()
         ans = " - ".join(ans["content"]) if "content" in ans else ""
-        if not ans:
-            return Crawler.be_output("")
+        if not is_valid_url(ans):
+            return Crawler.be_output("URL not valid")
         try:
             result = asyncio.run(self.get_web(ans))
 
@@ -64,7 +65,3 @@ class Crawler(ComponentBase, ABC):
             elif self._param.extract_type == 'content':
                 result.extracted_content
             return result.markdown
-            
-
-
-    

agent/component/deepl.py

@@ -14,7 +14,6 @@
 #  limitations under the License.
 #
 from abc import ABC
-import re
 from agent.component.base import ComponentBase, ComponentParamBase
 import deepl
 

agent/component/email.py

@@ -0,0 +1,141 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from abc import ABC
+import json
+import smtplib
+import logging
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from email.header import Header
+from email.utils import formataddr
+from agent.component.base import ComponentBase, ComponentParamBase
+
+class EmailParam(ComponentParamBase):
+    """
+    Define the Email component parameters.
+    """
+    def __init__(self):
+        super().__init__()
+        # Fixed configuration parameters
+        self.smtp_server = ""  # SMTP server address
+        self.smtp_port = 465  # SMTP port
+        self.email = ""  # Sender email
+        self.password = ""  # Email authorization code
+        self.sender_name = ""  # Sender name
+
+    def check(self):
+        # Check required parameters
+        self.check_empty(self.smtp_server, "SMTP Server")
+        self.check_empty(self.email, "Email")
+        self.check_empty(self.password, "Password")
+        self.check_empty(self.sender_name, "Sender Name")
+
+class Email(ComponentBase, ABC):
+    component_name = "Email"
+    
+    def _run(self, history, **kwargs):
+        # Get upstream component output and parse JSON
+        ans = self.get_input()
+        content = "".join(ans["content"]) if "content" in ans else ""
+        if not content:
+            return Email.be_output("No content to send")
+
+        success = False
+        try:
+            # Parse JSON string passed from upstream
+            email_data = json.loads(content)
+            
+            # Validate required fields
+            if "to_email" not in email_data:
+                return Email.be_output("Missing required field: to_email")
+
+            # Create email object
+            msg = MIMEMultipart('alternative')
+            
+            # Properly handle sender name encoding
+            msg['From'] = formataddr((str(Header(self._param.sender_name,'utf-8')), self._param.email))
+            msg['To'] = email_data["to_email"]
+            if "cc_email" in email_data and email_data["cc_email"]:
+                msg['Cc'] = email_data["cc_email"]
+            msg['Subject'] = Header(email_data.get("subject", "No Subject"), 'utf-8').encode()
+
+            # Use content from email_data or default content
+            email_content = email_data.get("content", "No content provided")
+            # msg.attach(MIMEText(email_content, 'plain', 'utf-8'))
+            msg.attach(MIMEText(email_content, 'html', 'utf-8'))
+
+            # Connect to SMTP server and send
+            logging.info(f"Connecting to SMTP server {self._param.smtp_server}:{self._param.smtp_port}")
+            
+            context = smtplib.ssl.create_default_context()
+            with smtplib.SMTP(self._param.smtp_server, self._param.smtp_port) as server:
+                server.ehlo()
+                server.starttls(context=context)
+                server.ehlo()
+                # Login
+                logging.info(f"Attempting to login with email: {self._param.email}")
+                server.login(self._param.email, self._param.password)
+                
+                # Get all recipient list
+                recipients = [email_data["to_email"]]
+                if "cc_email" in email_data and email_data["cc_email"]:
+                    recipients.extend(email_data["cc_email"].split(','))
+                
+                # Send email
+                logging.info(f"Sending email to recipients: {recipients}")
+                try:
+                    server.send_message(msg, self._param.email, recipients)
+                    success = True
+                except Exception as e:
+                    logging.error(f"Error during send_message: {str(e)}")
+                    # Try alternative method
+                    server.sendmail(self._param.email, recipients, msg.as_string())
+                    success = True
+                
+                try:
+                    server.quit()
+                except Exception as e:
+                    # Ignore errors when closing connection
+                    logging.warning(f"Non-fatal error during connection close: {str(e)}")
+
+            if success:
+                return Email.be_output("Email sent successfully")
+
+        except json.JSONDecodeError:
+            error_msg = "Invalid JSON format in input"
+            logging.error(error_msg)
+            return Email.be_output(error_msg)
+            
+        except smtplib.SMTPAuthenticationError:
+            error_msg = "SMTP Authentication failed. Please check your email and authorization code."
+            logging.error(error_msg)
+            return Email.be_output(f"Failed to send email: {error_msg}")
+            
+        except smtplib.SMTPConnectError:
+            error_msg = f"Failed to connect to SMTP server {self._param.smtp_server}:{self._param.smtp_port}"
+            logging.error(error_msg)
+            return Email.be_output(f"Failed to send email: {error_msg}")
+            
+        except smtplib.SMTPException as e:
+            error_msg = f"SMTP error occurred: {str(e)}"
+            logging.error(error_msg)
+            return Email.be_output(f"Failed to send email: {error_msg}")
+            
+        except Exception as e:
+            error_msg = f"Unexpected error: {str(e)}"
+            logging.error(error_msg)
+            return Email.be_output(f"Failed to send email: {error_msg}") 

agent/component/exesql.py

@@ -15,13 +15,17 @@
 #
 from abc import ABC
 import re
+from copy import deepcopy
+
 import pandas as pd
 import pymysql
 import psycopg2
-from agent.component.base import ComponentBase, ComponentParamBase
+from agent.component import GenerateParam, Generate
+import pyodbc
+import logging
 
 
-class ExeSQLParam(ComponentParamBase):
+class ExeSQLParam(GenerateParam):
     """
     Define the ExeSQL component parameters.
     """
@@ -38,7 +42,8 @@ class ExeSQLParam(ComponentParamBase):
         self.top_n = 30
 
     def check(self):
-        self.check_valid_value(self.db_type, "Choose DB type", ['mysql', 'postgresql', 'mariadb'])
+        super().check()
+        self.check_valid_value(self.db_type, "Choose DB type", ['mysql', 'postgresql', 'mariadb', 'mssql'])
         self.check_empty(self.database, "Database name")
         self.check_empty(self.username, "database username")
         self.check_empty(self.host, "IP Address")
@@ -46,58 +51,105 @@ class ExeSQLParam(ComponentParamBase):
         self.check_empty(self.password, "Database password")
         self.check_positive_integer(self.top_n, "Number of records")
         if self.database == "rag_flow":
-            if self.host == "ragflow-mysql": raise ValueError("The host is not accessible.")
-            if self.password == "infini_rag_flow": raise ValueError("The host is not accessible.")
+            if self.host == "ragflow-mysql":
+                raise ValueError("For the security reason, it dose not support database named rag_flow.")
+            if self.password == "infini_rag_flow":
+                raise ValueError("For the security reason, it dose not support database named rag_flow.")
 
 
-class ExeSQL(ComponentBase, ABC):
+class ExeSQL(Generate, ABC):
     component_name = "ExeSQL"
 
-    def _run(self, history, **kwargs):
-        if not hasattr(self, "_loop"):
-            setattr(self, "_loop", 0)
-        if self._loop >= self._param.loop:
-            self._loop = 0
-            raise Exception("Maximum loop time exceeds. Can't query the correct data via SQL statement.")
-        self._loop += 1
-
-        ans = self.get_input()
-        ans = "".join(ans["content"]) if "content" in ans else ""
-        ans = re.sub(r'^.*?SELECT ', 'SELECT ', repr(ans), flags=re.IGNORECASE)
+    def _refactor(self, ans):
+        ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+        match = re.search(r"```sql\s*(.*?)\s*```", ans, re.DOTALL)
+        if match:
+            ans = match.group(1)  # Query content
+            return ans
+        else:
+            print("no markdown")
+        ans = re.sub(r'^.*?SELECT ', 'SELECT ', (ans), flags=re.IGNORECASE)
         ans = re.sub(r';.*?SELECT ', '; SELECT ', ans, flags=re.IGNORECASE)
         ans = re.sub(r';[^;]*$', r';', ans)
         if not ans:
             raise Exception("SQL statement not found!")
+        return ans
 
+    def _run(self, history, **kwargs):
+        ans = self.get_input()
+        ans = "".join([str(a) for a in ans["content"]]) if "content" in ans else ""
+        ans = self._refactor(ans)
         if self._param.db_type in ["mysql", "mariadb"]:
             db = pymysql.connect(db=self._param.database, user=self._param.username, host=self._param.host,
                                  port=self._param.port, password=self._param.password)
         elif self._param.db_type == 'postgresql':
             db = psycopg2.connect(dbname=self._param.database, user=self._param.username, host=self._param.host,
                                   port=self._param.port, password=self._param.password)
-
+        elif self._param.db_type == 'mssql':
+            conn_str = (
+                    r'DRIVER={ODBC Driver 17 for SQL Server};'
+                    r'SERVER=' + self._param.host + ',' + str(self._param.port) + ';'
+                    r'DATABASE=' + self._param.database + ';'
+                    r'UID=' + self._param.username + ';'
+                    r'PWD=' + self._param.password
+            )
+            db = pyodbc.connect(conn_str)
         try:
             cursor = db.cursor()
         except Exception as e:
             raise Exception("Database Connection Failed! \n" + str(e))
+        if not hasattr(self, "_loop"):
+            setattr(self, "_loop", 0)
+            self._loop += 1
+        input_list = re.split(r';', ans.replace(r"\n", " "))
         sql_res = []
-        for single_sql in re.split(r';', ans.replace(r"\n", " ")):
-            if not single_sql:
-                continue
-            try:
-                cursor.execute(single_sql)
-                if cursor.rowcount == 0:
-                    sql_res.append({"content": "\nTotal: 0\n No record in the database!"})
-                    continue
-                single_res = pd.DataFrame([i for i in cursor.fetchmany(size=self._param.top_n)])
-                single_res.columns = [i[0] for i in cursor.description]
-                sql_res.append({"content": "\nTotal: " + str(cursor.rowcount) + "\n" + single_res.to_markdown()})
-            except Exception as e:
-                sql_res.append({"content": "**Error**:" + str(e) + "\nError SQL Statement:" + single_sql})
-                pass
+        for i in range(len(input_list)):
+            single_sql = input_list[i]
+            single_sql = single_sql.replace('```','')
+            while self._loop <= self._param.loop:
+                self._loop += 1
+                if not single_sql:
+                    break
+                try:
+                    cursor.execute(single_sql)
+                    if cursor.rowcount == 0:
+                        sql_res.append({"content": "No record in the database!"})
+                        break
+                    if self._param.db_type == 'mssql':
+                        single_res = pd.DataFrame.from_records(cursor.fetchmany(self._param.top_n),
+                                                               columns=[desc[0] for desc in cursor.description])
+                    else:
+                        single_res = pd.DataFrame([i for i in cursor.fetchmany(self._param.top_n)])
+                        single_res.columns = [i[0] for i in cursor.description]
+                    sql_res.append({"content": single_res.to_markdown(index=False, floatfmt=".6f")})
+                    break
+                except Exception as e:
+                    single_sql = self._regenerate_sql(single_sql, str(e), **kwargs)
+                    single_sql = self._refactor(single_sql)
+                    if self._loop > self._param.loop:
+                        sql_res.append({"content": "Can't query the correct data via SQL statement."})
         db.close()
-
         if not sql_res:
             return ExeSQL.be_output("")
-
         return pd.DataFrame(sql_res)
+
+    def _regenerate_sql(self, failed_sql, error_message, **kwargs):
+        prompt = f'''
+        ## You are the Repair SQL Statement Helper, please modify the original SQL statement based on the SQL query error report.
+        ## The original SQL statement is as follows:{failed_sql}.
+        ## The contents of the SQL query error report is as follows:{error_message}.
+        ## Answer only the modified SQL statement. Please do not give any explanation, just answer the code.
+'''
+        self._param.prompt = prompt
+        kwargs_ = deepcopy(kwargs)
+        kwargs_["stream"] = False
+        response = Generate._run(self, [], **kwargs_)
+        try:
+            regenerated_sql = response.loc[0, "content"]
+            return regenerated_sql
+        except Exception as e:
+            logging.error(f"Failed to regenerate SQL: {e}")
+            return None
+
+    def debug(self, **kwargs):
+        return self._run([], **kwargs)

agent/component/generate.py

@@ -13,14 +13,30 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import json
 import re
 from functools import partial
+from typing import Any
 import pandas as pd
 from api.db import LLMType
-from api.db.services.dialog_service import message_fit_in
+from api.db.services.conversation_service import structure_answer
 from api.db.services.llm_service import LLMBundle
 from api import settings
 from agent.component.base import ComponentBase, ComponentParamBase
+from plugin import GlobalPluginManager
+from plugin.llm_tool_plugin import llm_tool_metadata_to_openai_tool
+from rag.llm.chat_model import ToolCallSession
+from rag.prompts import message_fit_in
+
+
+class LLMToolPluginCallSession(ToolCallSession):
+    def tool_call(self, name: str, arguments: dict[str, Any]) -> str:
+        tool = GlobalPluginManager.get_llm_tool_by_name(name)
+
+        if tool is None:
+            raise ValueError(f"LLM tool {name} does not exist")
+
+        return tool().invoke(**arguments)
 
 
 class GenerateParam(ComponentParamBase):
@@ -39,6 +55,7 @@ class GenerateParam(ComponentParamBase):
         self.frequency_penalty = 0
         self.cite = True
         self.parameters = []
+        self.llm_enabled_tools = []
 
     def check(self):
         self.check_decimal_float(self.temperature, "[Generate] Temperature")
@@ -51,11 +68,16 @@ class GenerateParam(ComponentParamBase):
 
     def gen_conf(self):
         conf = {}
-        if self.max_tokens > 0: conf["max_tokens"] = self.max_tokens
-        if self.temperature > 0: conf["temperature"] = self.temperature
-        if self.top_p > 0: conf["top_p"] = self.top_p
-        if self.presence_penalty > 0: conf["presence_penalty"] = self.presence_penalty
-        if self.frequency_penalty > 0: conf["frequency_penalty"] = self.frequency_penalty
+        if self.max_tokens > 0:
+            conf["max_tokens"] = self.max_tokens
+        if self.temperature > 0:
+            conf["temperature"] = self.temperature
+        if self.top_p > 0:
+            conf["top_p"] = self.top_p
+        if self.presence_penalty > 0:
+            conf["presence_penalty"] = self.presence_penalty
+        if self.frequency_penalty > 0:
+            conf["frequency_penalty"] = self.frequency_penalty
         return conf
 
 
@@ -63,65 +85,96 @@ class Generate(ComponentBase):
     component_name = "Generate"
 
     def get_dependent_components(self):
-        cpnts = set([para["component_id"].split("@")[0] for para in self._param.parameters \
-                     if para.get("component_id") \
-                     and para["component_id"].lower().find("answer") < 0 \
-                     and para["component_id"].lower().find("begin") < 0])
+        inputs = self.get_input_elements()
+        cpnts = set([i["key"] for i in inputs[1:] if i["key"].lower().find("answer") < 0 and i["key"].lower().find("begin") < 0])
         return list(cpnts)
 
     def set_cite(self, retrieval_res, answer):
-        retrieval_res = retrieval_res.dropna(subset=["vector", "content_ltks"]).reset_index(drop=True)
         if "empty_response" in retrieval_res.columns:
             retrieval_res["empty_response"].fillna("", inplace=True)
+        chunks = json.loads(retrieval_res["chunks"][0])
         answer, idx = settings.retrievaler.insert_citations(answer,
-                                                            [ck["content_ltks"] for _, ck in retrieval_res.iterrows()],
-                                                            [ck["vector"] for _, ck in retrieval_res.iterrows()],
+                                                            [ck["content_ltks"] for ck in chunks],
+                                                            [ck["vector"] for ck in chunks],
                                                             LLMBundle(self._canvas.get_tenant_id(), LLMType.EMBEDDING,
                                                                       self._canvas.get_embedding_model()), tkweight=0.7,
                                                             vtweight=0.3)
         doc_ids = set([])
         recall_docs = []
         for i in idx:
-            did = retrieval_res.loc[int(i), "doc_id"]
-            if did in doc_ids: continue
+            did = chunks[int(i)]["doc_id"]
+            if did in doc_ids:
+                continue
             doc_ids.add(did)
-            recall_docs.append({"doc_id": did, "doc_name": retrieval_res.loc[int(i), "docnm_kwd"]})
+            recall_docs.append({"doc_id": did, "doc_name": chunks[int(i)]["docnm_kwd"]})
 
-        del retrieval_res["vector"]
-        del retrieval_res["content_ltks"]
+        for c in chunks:
+            del c["vector"]
+            del c["content_ltks"]
 
         reference = {
-            "chunks": [ck.to_dict() for _, ck in retrieval_res.iterrows()],
+            "chunks": chunks,
             "doc_aggs": recall_docs
         }
 
         if answer.lower().find("invalid key") >= 0 or answer.lower().find("invalid api") >= 0:
-            answer += " Please set LLM API-Key in 'User Setting -> Model Providers -> API-Key'"
+            answer += " Please set LLM API-Key in 'User Setting -> Model providers -> API-Key'"
         res = {"content": answer, "reference": reference}
+        res = structure_answer(None, res, "", "")
 
         return res
 
+    def get_input_elements(self):
+        key_set = set([])
+        res = [{"key": "user", "name": "Input your question here:"}]
+        for r in re.finditer(r"\{([a-z]+[:@][a-z0-9_-]+)\}", self._param.prompt, flags=re.IGNORECASE):
+            cpn_id = r.group(1)
+            if cpn_id in key_set:
+                continue
+            if cpn_id.lower().find("begin@") == 0:
+                cpn_id, key = cpn_id.split("@")
+                for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
+                    if p["key"] != key:
+                        continue
+                    res.append({"key": r.group(1), "name": p["name"]})
+                    key_set.add(r.group(1))
+                continue
+            cpn_nm = self._canvas.get_component_name(cpn_id)
+            if not cpn_nm:
+                continue
+            res.append({"key": cpn_id, "name": cpn_nm})
+            key_set.add(cpn_id)
+        return res
+
     def _run(self, history, **kwargs):
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
+
+        if len(self._param.llm_enabled_tools) > 0:
+            tools = GlobalPluginManager.get_llm_tools_by_names(self._param.llm_enabled_tools)
+
+            chat_mdl.bind_tools(
+                LLMToolPluginCallSession(),
+                [llm_tool_metadata_to_openai_tool(t.get_metadata()) for t in tools]
+            )
+
         prompt = self._param.prompt
 
         retrieval_res = []
         self._param.inputs = []
-        for para in self._param.parameters:
-            if not para.get("component_id"): continue
-            component_id = para["component_id"].split("@")[0]
-            if para["component_id"].lower().find("@") >= 0:
-                cpn_id, key = para["component_id"].split("@")
+        for para in self.get_input_elements()[1:]:
+            if para["key"].lower().find("begin@") == 0:
+                cpn_id, key = para["key"].split("@")
                 for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
                     if p["key"] == key:
                         kwargs[para["key"]] = p.get("value", "")
                         self._param.inputs.append(
-                            {"component_id": para["component_id"], "content": kwargs[para["key"]]})
+                            {"component_id": para["key"], "content": kwargs[para["key"]]})
                         break
                 else:
                     assert False, f"Can't find parameter '{key}' for {cpn_id}"
                 continue
 
+            component_id = para["key"]
             cpn = self._canvas.get_component(component_id)["obj"]
             if cpn.component_name.lower() == "answer":
                 hist = self._canvas.get_history(1)
@@ -137,12 +190,13 @@ class Generate(ComponentBase):
             else:
                 if cpn.component_name.lower() == "retrieval":
                     retrieval_res.append(out)
-                kwargs[para["key"]] = "  - "+"\n - ".join([o if isinstance(o, str) else str(o) for o in out["content"]])
-            self._param.inputs.append({"component_id": para["component_id"], "content": kwargs[para["key"]]})
+                kwargs[para["key"]] = "  - " + "\n - ".join([o if isinstance(o, str) else str(o) for o in out["content"]])
+            self._param.inputs.append({"component_id": para["key"], "content": kwargs[para["key"]]})
 
         if retrieval_res:
             retrieval_res = pd.concat(retrieval_res, ignore_index=True)
-        else: retrieval_res = pd.DataFrame([])
+        else:
+            retrieval_res = pd.DataFrame([])
 
         for n, v in kwargs.items():
             prompt = re.sub(r"\{%s\}" % re.escape(n), str(v).replace("\\", " "), prompt)
@@ -159,17 +213,20 @@ class Generate(ComponentBase):
             return partial(self.stream_output, chat_mdl, prompt, retrieval_res)
 
         if "empty_response" in retrieval_res.columns and not "".join(retrieval_res["content"]):
-            res = {"content": "\n- ".join(retrieval_res["empty_response"]) if "\n- ".join(
-                retrieval_res["empty_response"]) else "Nothing found in knowledgebase!", "reference": []}
+            empty_res = "\n- ".join([str(t) for t in retrieval_res["empty_response"] if str(t)])
+            res = {"content": empty_res if empty_res else "Nothing found in knowledgebase!", "reference": []}
             return pd.DataFrame([res])
 
         msg = self._canvas.get_history(self._param.message_history_window_size)
-        if len(msg) < 1: msg.append({"role": "user", "content": ""})
+        if len(msg) < 1:
+            msg.append({"role": "user", "content": "Output: "})
         _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(chat_mdl.max_length * 0.97))
-        if len(msg) < 2: msg.append({"role": "user", "content": ""})
+        if len(msg) < 2:
+            msg.append({"role": "user", "content": "Output: "})
         ans = chat_mdl.chat(msg[0]["content"], msg[1:], self._param.gen_conf())
-
-        if self._param.cite and "content_ltks" in retrieval_res.columns and "vector" in retrieval_res.columns:
+        ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+        self._canvas.set_component_infor(self._id, {"prompt":msg[0]["content"],"messages":  msg[1:],"conf":  self._param.gen_conf()})
+        if self._param.cite and "chunks" in retrieval_res.columns:
             res = self.set_cite(retrieval_res, ans)
             return pd.DataFrame([res])
 
@@ -178,24 +235,42 @@ class Generate(ComponentBase):
     def stream_output(self, chat_mdl, prompt, retrieval_res):
         res = None
         if "empty_response" in retrieval_res.columns and not "".join(retrieval_res["content"]):
-            res = {"content": "\n- ".join(retrieval_res["empty_response"]) if "\n- ".join(
-                retrieval_res["empty_response"]) else "Nothing found in knowledgebase!", "reference": []}
+            empty_res = "\n- ".join([str(t) for t in retrieval_res["empty_response"] if str(t)])
+            res = {"content": empty_res if empty_res else "Nothing found in knowledgebase!", "reference": []}
             yield res
             self.set_output(res)
             return
 
         msg = self._canvas.get_history(self._param.message_history_window_size)
-        if len(msg) < 1: msg.append({"role": "user", "content": ""})
+        if msg and msg[0]['role'] == 'assistant':
+            msg.pop(0)
+        if len(msg) < 1:
+            msg.append({"role": "user", "content": "Output: "})
         _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(chat_mdl.max_length * 0.97))
-        if len(msg) < 2: msg.append({"role": "user", "content": ""})
+        if len(msg) < 2:
+            msg.append({"role": "user", "content": "Output: "})
         answer = ""
         for ans in chat_mdl.chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf()):
             res = {"content": ans, "reference": []}
             answer = ans
             yield res
 
-        if self._param.cite and "content_ltks" in retrieval_res.columns and "vector" in retrieval_res.columns:
+        if self._param.cite and "chunks" in retrieval_res.columns:
             res = self.set_cite(retrieval_res, answer)
             yield res
+        self._canvas.set_component_infor(self._id, {"prompt":msg[0]["content"],"messages":  msg[1:],"conf":  self._param.gen_conf()})
+        self.set_output(Generate.be_output(res))
 
-        self.set_output(res)
+    def debug(self, **kwargs):
+        chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
+        prompt = self._param.prompt
+
+        for para in self._param.debug_inputs:
+            kwargs[para["key"]] = para.get("value", "")
+
+        for n, v in kwargs.items():
+            prompt = re.sub(r"\{%s\}" % re.escape(n), str(v).replace("\\", " "), prompt)
+
+        u = kwargs.get("user")
+        ans = chat_mdl.chat(prompt, [{"role": "user", "content": u if u else "Output: "}], self._param.gen_conf())
+        return pd.DataFrame([ans])

agent/component/invoke.py

@@ -35,12 +35,14 @@ class InvokeParam(ComponentParamBase):
         self.url = ""
         self.timeout = 60
         self.clean_html = False
+        self.datatype = "json"  # New parameter to determine data posting type
 
     def check(self):
         self.check_valid_value(self.method.lower(), "Type of content from the crawler", ['get', 'post', 'put'])
         self.check_empty(self.url, "End point URL")
         self.check_positive_integer(self.timeout, "Timeout time in second")
         self.check_boolean(self.clean_html, "Clean HTML")
+        self.check_valid_value(self.datatype.lower(), "Data post type", ['json', 'formdata'])  # Check for valid datapost value
 
 
 class Invoke(ComponentBase, ABC):
@@ -50,14 +52,24 @@ class Invoke(ComponentBase, ABC):
         args = {}
         for para in self._param.variables:
             if para.get("component_id"):
-                cpn = self._canvas.get_component(para["component_id"])["obj"]
-                if cpn.component_name.lower() == "answer":
-                    args[para["key"]] = self._canvas.get_history(1)[0]["content"]
-                    continue
-                _, out = cpn.output(allow_partial=False)
-                args[para["key"]] = "\n".join(out["content"])
+                if '@' in para["component_id"]:
+                    component = para["component_id"].split('@')[0]
+                    field = para["component_id"].split('@')[1]
+                    cpn = self._canvas.get_component(component)["obj"]
+                    for param in cpn._param.query:
+                        if param["key"] == field:
+                            if "value" in param:
+                                args[para["key"]] = param["value"]
+                else:
+                    cpn = self._canvas.get_component(para["component_id"])["obj"]
+                    if cpn.component_name.lower() == "answer":
+                        args[para["key"]] = self._canvas.get_history(1)[0]["content"]
+                        continue
+                    _, out = cpn.output(allow_partial=False)
+                    if not out.empty:
+                        args[para["key"]] = "\n".join(out["content"])
             else:
-                args[para["key"]] = "\n".join(para["value"])
+                args[para["key"]] = para["value"]
 
         url = self._param.url.strip()
         if url.find("http") != 0:
@@ -84,22 +96,36 @@ class Invoke(ComponentBase, ABC):
             return Invoke.be_output(response.text)
 
         if method == 'put':
-            response = requests.put(url=url,
-                                    data=args,
-                                    headers=headers,
-                                    proxies=proxies,
-                                    timeout=self._param.timeout)
+            if self._param.datatype.lower() == 'json':
+                response = requests.put(url=url,
+                                        json=args,
+                                        headers=headers,
+                                        proxies=proxies,
+                                        timeout=self._param.timeout)
+            else:
+                response = requests.put(url=url,
+                                        data=args,
+                                        headers=headers,
+                                        proxies=proxies,
+                                        timeout=self._param.timeout)
             if self._param.clean_html:
                 sections = HtmlParser()(None, response.content)
                 return Invoke.be_output("\n".join(sections))
             return Invoke.be_output(response.text)
 
         if method == 'post':
-            response = requests.post(url=url,
-                                    json=args,
-                                    headers=headers,
-                                    proxies=proxies,
-                                    timeout=self._param.timeout)
+            if self._param.datatype.lower() == 'json':
+                response = requests.post(url=url,
+                                         json=args,
+                                         headers=headers,
+                                         proxies=proxies,
+                                         timeout=self._param.timeout)
+            else:
+                response = requests.post(url=url,
+                                         data=args,
+                                         headers=headers,
+                                         proxies=proxies,
+                                         timeout=self._param.timeout)
             if self._param.clean_html:
                 sections = HtmlParser()(None, response.content)
                 return Invoke.be_output("\n".join(sections))

agent/component/iteration.py

@@ -0,0 +1,45 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from abc import ABC
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class IterationParam(ComponentParamBase):
+    """
+    Define the Iteration component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.delimiter = ","
+
+    def check(self):
+        self.check_empty(self.delimiter, "Delimiter")
+
+
+class Iteration(ComponentBase, ABC):
+    component_name = "Iteration"
+
+    def get_start(self):
+        for cid in self._canvas.components.keys():
+            if self._canvas.get_component(cid)["obj"].component_name.lower() != "iterationitem":
+                continue
+            if self._canvas.get_component(cid)["parent_id"] == self._id:
+                return self._canvas.get_component(cid)
+
+    def _run(self, history, **kwargs):
+        return self.output(allow_partial=False)[1]
+

agent/component/iterationitem.py

@@ -0,0 +1,53 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from abc import ABC
+import pandas as pd
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class IterationItemParam(ComponentParamBase):
+    """
+    Define the IterationItem component parameters.
+    """
+    def check(self):
+        return True
+
+
+class IterationItem(ComponentBase, ABC):
+    component_name = "IterationItem"
+
+    def __init__(self, canvas, id, param: ComponentParamBase):
+        super().__init__(canvas, id, param)
+        self._idx = 0
+
+    def _run(self, history, **kwargs):
+        parent = self.get_parent()
+        ans = parent.get_input()
+        ans = parent._param.delimiter.join(ans["content"]) if "content" in ans else ""
+        ans = [a.strip() for a in ans.split(parent._param.delimiter)]
+        if not ans:
+            self._idx = -1
+            return pd.DataFrame()
+
+        df = pd.DataFrame([{"content": ans[self._idx]}])
+        self._idx += 1
+        if self._idx >= len(ans):
+            self._idx = -1
+        return df
+
+    def end(self):
+        return self._idx == -1
+

agent/component/jin10.py

@@ -1,130 +1,130 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-import json
-from abc import ABC
-import pandas as pd
-import requests
-from agent.component.base import ComponentBase, ComponentParamBase
-
-
-class Jin10Param(ComponentParamBase):
-    """
-    Define the Jin10 component parameters.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.type = "flash"
-        self.secret_key = "xxx"
-        self.flash_type = '1'
-        self.calendar_type = 'cj'
-        self.calendar_datatype = 'data'
-        self.symbols_type = 'GOODS'
-        self.symbols_datatype = 'symbols'
-        self.contain = ""
-        self.filter = ""
-
-    def check(self):
-        self.check_valid_value(self.type, "Type", ['flash', 'calendar', 'symbols', 'news'])
-        self.check_valid_value(self.flash_type, "Flash Type", ['1', '2', '3', '4', '5'])
-        self.check_valid_value(self.calendar_type, "Calendar Type", ['cj', 'qh', 'hk', 'us'])
-        self.check_valid_value(self.calendar_datatype, "Calendar DataType", ['data', 'event', 'holiday'])
-        self.check_valid_value(self.symbols_type, "Symbols Type", ['GOODS', 'FOREX', 'FUTURE', 'CRYPTO'])
-        self.check_valid_value(self.symbols_datatype, 'Symbols DataType', ['symbols', 'quotes'])
-
-
-class Jin10(ComponentBase, ABC):
-    component_name = "Jin10"
-
-    def _run(self, history, **kwargs):
-        ans = self.get_input()
-        ans = " - ".join(ans["content"]) if "content" in ans else ""
-        if not ans:
-            return Jin10.be_output("")
-
-        jin10_res = []
-        headers = {'secret-key': self._param.secret_key}
-        try:
-            if self._param.type == "flash":
-                params = {
-                    'category': self._param.flash_type,
-                    'contain': self._param.contain,
-                    'filter': self._param.filter
-                }
-                response = requests.get(
-                    url='https://open-data-api.jin10.com/data-api/flash?category=' + self._param.flash_type,
-                    headers=headers, data=json.dumps(params))
-                response = response.json()
-                for i in response['data']:
-                    jin10_res.append({"content": i['data']['content']})
-            if self._param.type == "calendar":
-                params = {
-                    'category': self._param.calendar_type
-                }
-                response = requests.get(
-                    url='https://open-data-api.jin10.com/data-api/calendar/' + self._param.calendar_datatype + '?category=' + self._param.calendar_type,
-                    headers=headers, data=json.dumps(params))
-
-                response = response.json()
-                jin10_res.append({"content": pd.DataFrame(response['data']).to_markdown()})
-            if self._param.type == "symbols":
-                params = {
-                    'type': self._param.symbols_type
-                }
-                if self._param.symbols_datatype == "quotes":
-                    params['codes'] = 'BTCUSD'
-                response = requests.get(
-                    url='https://open-data-api.jin10.com/data-api/' + self._param.symbols_datatype + '?type=' + self._param.symbols_type,
-                    headers=headers, data=json.dumps(params))
-                response = response.json()
-                if self._param.symbols_datatype == "symbols":
-                    for i in response['data']:
-                        i['Commodity Code'] = i['c']
-                        i['Stock Exchange'] = i['e']
-                        i['Commodity Name'] = i['n']
-                        i['Commodity Type'] = i['t']
-                        del i['c'], i['e'], i['n'], i['t']
-                if self._param.symbols_datatype == "quotes":
-                    for i in response['data']:
-                        i['Selling Price'] = i['a']
-                        i['Buying Price'] = i['b']
-                        i['Commodity Code'] = i['c']
-                        i['Stock Exchange'] = i['e']
-                        i['Highest Price'] = i['h']
-                        i['Yesterday’s Closing Price'] = i['hc']
-                        i['Lowest Price'] = i['l']
-                        i['Opening Price'] = i['o']
-                        i['Latest Price'] = i['p']
-                        i['Market Quote Time'] = i['t']
-                        del i['a'], i['b'], i['c'], i['e'], i['h'], i['hc'], i['l'], i['o'], i['p'], i['t']
-                jin10_res.append({"content": pd.DataFrame(response['data']).to_markdown()})
-            if self._param.type == "news":
-                params = {
-                    'contain': self._param.contain,
-                    'filter': self._param.filter
-                }
-                response = requests.get(
-                    url='https://open-data-api.jin10.com/data-api/news',
-                    headers=headers, data=json.dumps(params))
-                response = response.json()
-                jin10_res.append({"content": pd.DataFrame(response['data']).to_markdown()})
-        except Exception as e:
-            return Jin10.be_output("**ERROR**: " + str(e))
-
-        if not jin10_res:
-            return Jin10.be_output("")
-
-        return pd.DataFrame(jin10_res)
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import json
+from abc import ABC
+import pandas as pd
+import requests
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class Jin10Param(ComponentParamBase):
+    """
+    Define the Jin10 component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.type = "flash"
+        self.secret_key = "xxx"
+        self.flash_type = '1'
+        self.calendar_type = 'cj'
+        self.calendar_datatype = 'data'
+        self.symbols_type = 'GOODS'
+        self.symbols_datatype = 'symbols'
+        self.contain = ""
+        self.filter = ""
+
+    def check(self):
+        self.check_valid_value(self.type, "Type", ['flash', 'calendar', 'symbols', 'news'])
+        self.check_valid_value(self.flash_type, "Flash Type", ['1', '2', '3', '4', '5'])
+        self.check_valid_value(self.calendar_type, "Calendar Type", ['cj', 'qh', 'hk', 'us'])
+        self.check_valid_value(self.calendar_datatype, "Calendar DataType", ['data', 'event', 'holiday'])
+        self.check_valid_value(self.symbols_type, "Symbols Type", ['GOODS', 'FOREX', 'FUTURE', 'CRYPTO'])
+        self.check_valid_value(self.symbols_datatype, 'Symbols DataType', ['symbols', 'quotes'])
+
+
+class Jin10(ComponentBase, ABC):
+    component_name = "Jin10"
+
+    def _run(self, history, **kwargs):
+        ans = self.get_input()
+        ans = " - ".join(ans["content"]) if "content" in ans else ""
+        if not ans:
+            return Jin10.be_output("")
+
+        jin10_res = []
+        headers = {'secret-key': self._param.secret_key}
+        try:
+            if self._param.type == "flash":
+                params = {
+                    'category': self._param.flash_type,
+                    'contain': self._param.contain,
+                    'filter': self._param.filter
+                }
+                response = requests.get(
+                    url='https://open-data-api.jin10.com/data-api/flash?category=' + self._param.flash_type,
+                    headers=headers, data=json.dumps(params))
+                response = response.json()
+                for i in response['data']:
+                    jin10_res.append({"content": i['data']['content']})
+            if self._param.type == "calendar":
+                params = {
+                    'category': self._param.calendar_type
+                }
+                response = requests.get(
+                    url='https://open-data-api.jin10.com/data-api/calendar/' + self._param.calendar_datatype + '?category=' + self._param.calendar_type,
+                    headers=headers, data=json.dumps(params))
+
+                response = response.json()
+                jin10_res.append({"content": pd.DataFrame(response['data']).to_markdown()})
+            if self._param.type == "symbols":
+                params = {
+                    'type': self._param.symbols_type
+                }
+                if self._param.symbols_datatype == "quotes":
+                    params['codes'] = 'BTCUSD'
+                response = requests.get(
+                    url='https://open-data-api.jin10.com/data-api/' + self._param.symbols_datatype + '?type=' + self._param.symbols_type,
+                    headers=headers, data=json.dumps(params))
+                response = response.json()
+                if self._param.symbols_datatype == "symbols":
+                    for i in response['data']:
+                        i['Commodity Code'] = i['c']
+                        i['Stock Exchange'] = i['e']
+                        i['Commodity Name'] = i['n']
+                        i['Commodity Type'] = i['t']
+                        del i['c'], i['e'], i['n'], i['t']
+                if self._param.symbols_datatype == "quotes":
+                    for i in response['data']:
+                        i['Selling Price'] = i['a']
+                        i['Buying Price'] = i['b']
+                        i['Commodity Code'] = i['c']
+                        i['Stock Exchange'] = i['e']
+                        i['Highest Price'] = i['h']
+                        i['Yesterday’s Closing Price'] = i['hc']
+                        i['Lowest Price'] = i['l']
+                        i['Opening Price'] = i['o']
+                        i['Latest Price'] = i['p']
+                        i['Market Quote Time'] = i['t']
+                        del i['a'], i['b'], i['c'], i['e'], i['h'], i['hc'], i['l'], i['o'], i['p'], i['t']
+                jin10_res.append({"content": pd.DataFrame(response['data']).to_markdown()})
+            if self._param.type == "news":
+                params = {
+                    'contain': self._param.contain,
+                    'filter': self._param.filter
+                }
+                response = requests.get(
+                    url='https://open-data-api.jin10.com/data-api/news',
+                    headers=headers, data=json.dumps(params))
+                response = response.json()
+                jin10_res.append({"content": pd.DataFrame(response['data']).to_markdown()})
+        except Exception as e:
+            return Jin10.be_output("**ERROR**: " + str(e))
+
+        if not jin10_res:
+            return Jin10.be_output("")
+
+        return pd.DataFrame(jin10_res)

agent/component/keyword.py

@@ -51,12 +51,22 @@ class KeywordExtract(Generate, ABC):
 
     def _run(self, history, **kwargs):
         query = self.get_input()
-        query = str(query["content"][0]) if "content" in query else ""
+        if hasattr(query, "to_dict") and "content" in query:
+            query = ", ".join(map(str, query["content"].dropna()))
+        else:
+            query = str(query)
+
 
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
+        self._canvas.set_component_infor(self._id, {"prompt":self._param.get_prompt(),"messages":  [{"role": "user", "content": query}],"conf": self._param.gen_conf()})
+
         ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": query}],
                             self._param.gen_conf())
 
+        ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
         ans = re.sub(r".*keyword:", "", ans).strip()
         logging.debug(f"ans: {ans}")
         return KeywordExtract.be_output(ans)
+
+    def debug(self, **kwargs):
+        return self._run([], **kwargs)

agent/component/relevant.py

@@ -78,4 +78,6 @@ class Relevant(Generate, ABC):
             return Relevant.be_output(self._param.no)
         assert False, f"Relevant component got: {ans}"
 
+    def debug(self, **kwargs):
+        return self._run([], **kwargs)
 

agent/component/retrieval.py

@@ -13,7 +13,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import json
 import logging
+import re
 from abc import ABC
 
 import pandas as pd
@@ -23,13 +25,16 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api import settings
 from agent.component.base import ComponentBase, ComponentParamBase
+from rag.app.tag import label_question
+from rag.prompts import kb_prompt
+from rag.utils.tavily_conn import Tavily
 
 
 class RetrievalParam(ComponentParamBase):
-
     """
     Define the Retrieval component parameters.
     """
+
     def __init__(self):
         super().__init__()
         self.similarity_threshold = 0.2
@@ -37,12 +42,15 @@ class RetrievalParam(ComponentParamBase):
         self.top_n = 8
         self.top_k = 1024
         self.kb_ids = []
+        self.kb_vars = []
         self.rerank_id = ""
         self.empty_response = ""
+        self.tavily_api_key = ""
+        self.use_kg = False
 
     def check(self):
         self.check_decimal_float(self.similarity_threshold, "[Retrieval] Similarity threshold")
-        self.check_decimal_float(self.keywords_similarity_weight, "[Retrieval] Keywords similarity weight")
+        self.check_decimal_float(self.keywords_similarity_weight, "[Retrieval] Keyword similarity weight")
         self.check_positive_number(self.top_n, "[Retrieval] Top N")
 
 
@@ -52,25 +60,68 @@ class Retrieval(ComponentBase, ABC):
     def _run(self, history, **kwargs):
         query = self.get_input()
         query = str(query["content"][0]) if "content" in query else ""
+        query = re.split(r"(USER:|ASSISTANT:)", query)[-1]
 
-        kbs = KnowledgebaseService.get_by_ids(self._param.kb_ids)
+        kb_ids: list[str] = self._param.kb_ids or []
+
+        kb_vars = self._fetch_outputs_from(self._param.kb_vars)
+
+        if len(kb_vars) > 0:
+            for kb_var in kb_vars:
+                if len(kb_var) == 1:
+                    kb_var_value = str(kb_var["content"][0])
+
+                    for v in kb_var_value.split(","):
+                        kb_ids.append(v)
+                else:
+                    for v in kb_var.to_dict("records"):
+                        kb_ids.append(v["content"])
+
+        filtered_kb_ids: list[str] = [kb_id for kb_id in kb_ids if kb_id]
+
+        kbs = KnowledgebaseService.get_by_ids(filtered_kb_ids)
         if not kbs:
             return Retrieval.be_output("")
 
         embd_nms = list(set([kb.embd_id for kb in kbs]))
         assert len(embd_nms) == 1, "Knowledge bases use different embedding models."
 
-        embd_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.EMBEDDING, embd_nms[0])
-        self._canvas.set_embedding_model(embd_nms[0])
+        embd_mdl = None
+        if embd_nms:
+            embd_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.EMBEDDING, embd_nms[0])
+            self._canvas.set_embedding_model(embd_nms[0])
 
         rerank_mdl = None
         if self._param.rerank_id:
             rerank_mdl = LLMBundle(kbs[0].tenant_id, LLMType.RERANK, self._param.rerank_id)
 
-        kbinfos = settings.retrievaler.retrieval(query, embd_mdl, kbs[0].tenant_id, self._param.kb_ids,
-                                        1, self._param.top_n,
-                                        self._param.similarity_threshold, 1 - self._param.keywords_similarity_weight,
-                                        aggs=False, rerank_mdl=rerank_mdl)
+        if kbs:
+            kbinfos = settings.retrievaler.retrieval(
+                query,
+                embd_mdl,
+                [kb.tenant_id for kb in kbs],
+                filtered_kb_ids,
+                1,
+                self._param.top_n,
+                self._param.similarity_threshold,
+                1 - self._param.keywords_similarity_weight,
+                aggs=False,
+                rerank_mdl=rerank_mdl,
+                rank_feature=label_question(query, kbs),
+            )
+        else:
+            kbinfos = {"chunks": [], "doc_aggs": []}
+
+        if self._param.use_kg and kbs:
+            ck = settings.kg_retrievaler.retrieval(query, [kb.tenant_id for kb in kbs], filtered_kb_ids, embd_mdl, LLMBundle(kbs[0].tenant_id, LLMType.CHAT))
+            if ck["content_with_weight"]:
+                kbinfos["chunks"].insert(0, ck)
+
+        if self._param.tavily_api_key:
+            tav = Tavily(self._param.tavily_api_key)
+            tav_res = tav.retrieve_chunks(query)
+            kbinfos["chunks"].extend(tav_res["chunks"])
+            kbinfos["doc_aggs"].extend(tav_res["doc_aggs"])
 
         if not kbinfos["chunks"]:
             df = Retrieval.be_output("")
@@ -78,10 +129,6 @@ class Retrieval(ComponentBase, ABC):
                 df["empty_response"] = self._param.empty_response
             return df
 
-        df = pd.DataFrame(kbinfos["chunks"])
-        df["content"] = df["content_with_weight"]
-        del df["content_with_weight"]
+        df = pd.DataFrame({"content": kb_prompt(kbinfos, 200000), "chunks": json.dumps(kbinfos["chunks"])})
         logging.debug("{} {}".format(query, df))
-        return df
-
-
+        return df.dropna()

agent/component/rewrite.py

@@ -13,99 +13,82 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import logging
 from abc import ABC
-from api.db import LLMType
-from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
+from rag.prompts import full_question
 
 
 class RewriteQuestionParam(GenerateParam):
-
     """
     Define the QuestionRewrite component parameters.
     """
+
     def __init__(self):
         super().__init__()
         self.temperature = 0.9
         self.prompt = ""
-        self.loop = 1
+        self.language = ""
 
     def check(self):
         super().check()
 
-    def get_prompt(self, conv):
-        self.prompt = """
-        You are an expert at query expansion to generate a paraphrasing of a question.
-        I can't retrieval relevant information from the knowledge base by using user's question directly.     
-        You need to expand or paraphrase user's question by multiple ways such as using synonyms words/phrase, 
-        writing the abbreviation in its entirety, adding some extra descriptions or explanations, 
-        changing the way of expression, translating the original question into another language (English/Chinese), etc. 
-        And return 5 versions of question and one is from translation.
-        Just list the question. No other words are needed.
-        """
-        return f"""
-Role: A helpful assistant
-Task: Generate a full user question that would follow the conversation.
-Requirements & Restrictions:
-  - Text generated MUST be in the same language of the original user's question.
-  - If the user's latest question is completely, don't do anything, just return the original question.
-  - DON'T generate anything except a refined question.
-
-######################
--Examples-
-######################
-# Example 1
-## Conversation
-USER: What is the name of Donald Trump's father?
-ASSISTANT:  Fred Trump.
-USER: And his mother?
-###############
-Output: What's the name of Donald Trump's mother?
-------------
-# Example 2
-## Conversation
-USER: What is the name of Donald Trump's father?
-ASSISTANT:  Fred Trump.
-USER: And his mother?
-ASSISTANT:  Mary Trump.
-User: What's her full name?
-###############
-Output: What's the full name of Donald Trump's mother Mary Trump?
-######################
-# Real Data
-## Conversation
-{conv}
-###############
-    """
-        return self.prompt
-
 
 class RewriteQuestion(Generate, ABC):
     component_name = "RewriteQuestion"
 
     def _run(self, history, **kwargs):
-        if not hasattr(self, "_loop"):
-            setattr(self, "_loop", 0)
-        if self._loop >= self._param.loop:
-            self._loop = 0
-            raise Exception("Sorry! Nothing relevant found.")
-        self._loop += 1
-
-        hist = self._canvas.get_history(4)
-        conv = []
-        for m in hist:
-            if m["role"] not in ["user", "assistant"]: continue
-            conv.append("{}: {}".format(m["role"].upper(), m["content"]))
-        conv = "\n".join(conv)
-
-        chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
-        ans = chat_mdl.chat(self._param.get_prompt(conv), [{"role": "user", "content": "Output: "}],
-                            self._param.gen_conf())
+        hist = self._canvas.get_history(self._param.message_history_window_size)
+        query = self.get_input()
+        query = str(query["content"][0]) if "content" in query else ""
+        messages = [h for h in hist if h["role"]!="system"]
+        if messages[-1]["role"] != "user":
+            messages.append({"role": "user", "content": query})
+        ans = full_question(self._canvas.get_tenant_id(), self._param.llm_id, messages, self.gen_lang(self._param.language))
         self._canvas.history.pop()
         self._canvas.history.append(("user", ans))
-
-        logging.debug(ans)
         return RewriteQuestion.be_output(ans)
 
-
+    @staticmethod
+    def gen_lang(language):
+        # convert code lang to language word for the prompt
+        language_dict = {'af': 'Afrikaans', 'ak': 'Akan', 'sq': 'Albanian', 'ws': 'Samoan', 'am': 'Amharic',
+                         'ar': 'Arabic', 'hy': 'Armenian', 'az': 'Azerbaijani', 'eu': 'Basque', 'be': 'Belarusian',
+                         'bem': 'Bemba', 'bn': 'Bengali', 'bh': 'Bihari',
+                         'xx-bork': 'Bork', 'bs': 'Bosnian', 'br': 'Breton', 'bg': 'Bulgarian', 'bt': 'Bhutani',
+                         'km': 'Cambodian', 'ca': 'Catalan', 'chr': 'Cherokee', 'ny': 'Chichewa', 'zh-cn': 'Chinese',
+                         'zh-tw': 'Chinese', 'co': 'Corsican',
+                         'hr': 'Croatian', 'cs': 'Czech', 'da': 'Danish', 'nl': 'Dutch', 'xx-elmer': 'Elmer',
+                         'en': 'English', 'eo': 'Esperanto', 'et': 'Estonian', 'ee': 'Ewe', 'fo': 'Faroese',
+                         'tl': 'Filipino', 'fi': 'Finnish', 'fr': 'French',
+                         'fy': 'Frisian', 'gaa': 'Ga', 'gl': 'Galician', 'ka': 'Georgian', 'de': 'German',
+                         'el': 'Greek', 'kl': 'Greenlandic', 'gn': 'Guarani', 'gu': 'Gujarati', 'xx-hacker': 'Hacker',
+                         'ht': 'Haitian Creole', 'ha': 'Hausa', 'haw': 'Hawaiian',
+                         'iw': 'Hebrew', 'hi': 'Hindi', 'hu': 'Hungarian', 'is': 'Icelandic', 'ig': 'Igbo',
+                         'id': 'Indonesian', 'ia': 'Interlingua', 'ga': 'Irish', 'it': 'Italian', 'ja': 'Japanese',
+                         'jw': 'Javanese', 'kn': 'Kannada', 'kk': 'Kazakh', 'rw': 'Kinyarwanda',
+                         'rn': 'Kirundi', 'xx-klingon': 'Klingon', 'kg': 'Kongo', 'ko': 'Korean', 'kri': 'Krio',
+                         'ku': 'Kurdish', 'ckb': 'Kurdish (Sorani)', 'ky': 'Kyrgyz', 'lo': 'Laothian', 'la': 'Latin',
+                         'lv': 'Latvian', 'ln': 'Lingala', 'lt': 'Lithuanian',
+                         'loz': 'Lozi', 'lg': 'Luganda', 'ach': 'Luo', 'mk': 'Macedonian', 'mg': 'Malagasy',
+                         'ms': 'Malay', 'ml': 'Malayalam', 'mt': 'Maltese', 'mv': 'Maldivian', 'mi': 'Maori',
+                         'mr': 'Marathi', 'mfe': 'Mauritian Creole', 'mo': 'Moldavian', 'mn': 'Mongolian',
+                         'sr-me': 'Montenegrin', 'my': 'Burmese', 'ne': 'Nepali', 'pcm': 'Nigerian Pidgin',
+                         'nso': 'Northern Sotho', 'no': 'Norwegian', 'nn': 'Norwegian Nynorsk', 'oc': 'Occitan',
+                         'or': 'Oriya', 'om': 'Oromo', 'ps': 'Pashto', 'fa': 'Persian',
+                         'xx-pirate': 'Pirate', 'pl': 'Polish', 'pt': 'Portuguese', 'pt-br': 'Portuguese (Brazilian)',
+                         'pt-pt': 'Portuguese (Portugal)', 'pa': 'Punjabi', 'qu': 'Quechua', 'ro': 'Romanian',
+                         'rm': 'Romansh', 'nyn': 'Runyankole', 'ru': 'Russian', 'gd': 'Scots Gaelic',
+                         'sr': 'Serbian', 'sh': 'Serbo-Croatian', 'st': 'Sesotho', 'tn': 'Setswana',
+                         'crs': 'Seychellois Creole', 'sn': 'Shona', 'sd': 'Sindhi', 'si': 'Sinhalese', 'sk': 'Slovak',
+                         'sl': 'Slovenian', 'so': 'Somali', 'es': 'Spanish', 'es-419': 'Spanish (Latin America)',
+                         'su': 'Sundanese',
+                         'sw': 'Swahili', 'sv': 'Swedish', 'tg': 'Tajik', 'ta': 'Tamil', 'tt': 'Tatar', 'te': 'Telugu',
+                         'th': 'Thai', 'ti': 'Tigrinya', 'to': 'Tongan', 'lua': 'Tshiluba', 'tum': 'Tumbuka',
+                         'tr': 'Turkish', 'tk': 'Turkmen', 'tw': 'Twi',
+                         'ug': 'Uyghur', 'uk': 'Ukrainian', 'ur': 'Urdu', 'uz': 'Uzbek', 'vu': 'Vanuatu',
+                         'vi': 'Vietnamese', 'cy': 'Welsh', 'wo': 'Wolof', 'xh': 'Xhosa', 'yi': 'Yiddish',
+                         'yo': 'Yoruba', 'zu': 'Zulu'}
+        if language in language_dict:
+            return language_dict[language]
+        else:
+            return ""

agent/component/switch.py

@@ -41,7 +41,8 @@ class SwitchParam(ComponentParamBase):
     def check(self):
         self.check_empty(self.conditions, "[Switch] conditions")
         for cond in self.conditions:
-            if not cond["to"]: raise ValueError(f"[Switch] 'To' can not be empty!")
+            if not cond["to"]:
+                raise ValueError("[Switch] 'To' can not be empty!")
 
 
 class Switch(ComponentBase, ABC):
@@ -51,8 +52,9 @@ class Switch(ComponentBase, ABC):
         res = []
         for cond in self._param.conditions:
             for item in cond["items"]:
-                if not item["cpn_id"]: continue
-                if item["cpn_id"].find("begin") >= 0:
+                if not item["cpn_id"]:
+                    continue
+                if item["cpn_id"].lower().find("begin") >= 0 or item["cpn_id"].lower().find("answer") >= 0:
                     continue
                 cid = item["cpn_id"].split("@")[0]
                 res.append(cid)
@@ -63,7 +65,8 @@ class Switch(ComponentBase, ABC):
         for cond in self._param.conditions:
             res = []
             for item in cond["items"]:
-                if not item["cpn_id"]:continue
+                if not item["cpn_id"]:
+                    continue
                 cid = item["cpn_id"].split("@")[0]
                 if item["cpn_id"].find("@") > 0:
                     cpn_id, key = item["cpn_id"].split("@")
@@ -72,7 +75,7 @@ class Switch(ComponentBase, ABC):
                             res.append(self.process_operator(p.get("value",""), item["operator"], item.get("value", "")))
                             break
                 else:
-                    out = self._canvas.get_component(cid)["obj"].output()[1]
+                    out = self._canvas.get_component(cid)["obj"].output(allow_partial=False)[1]
                     cpn_input = "" if "content" not in out.columns else " ".join([str(s) for s in out["content"]])
                     res.append(self.process_operator(cpn_input, item["operator"], item.get("value", "")))
 
@@ -107,22 +110,22 @@ class Switch(ComponentBase, ABC):
         elif operator == ">":
             try:
                 return True if float(input) > float(value) else False
-            except Exception as e:
+            except Exception:
                 return True if input > value else False
         elif operator == "<":
             try:
                 return True if float(input) < float(value) else False
-            except Exception as e:
+            except Exception:
                 return True if input < value else False
         elif operator == "≥":
             try:
                 return True if float(input) >= float(value) else False
-            except Exception as e:
+            except Exception:
                 return True if input >= value else False
         elif operator == "≤":
             try:
                 return True if float(input) <= float(value) else False
-            except Exception as e:
+            except Exception:
                 return True if input <= value else False
 
         raise ValueError('Not supported operator' + operator)

agent/component/template.py

@@ -13,8 +13,10 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import json
 import re
 from agent.component.base import ComponentBase, ComponentParamBase
+from jinja2 import Template as Jinja2Template
 
 
 class TemplateParam(ComponentParamBase):
@@ -36,31 +38,49 @@ class Template(ComponentBase):
     component_name = "Template"
 
     def get_dependent_components(self):
-        cpnts = set([para["component_id"].split("@")[0] for para in self._param.parameters \
-                     if para.get("component_id") \
-                     and para["component_id"].lower().find("answer") < 0 \
-                     and para["component_id"].lower().find("begin") < 0])
+        inputs = self.get_input_elements()
+        cpnts = set([i["key"] for i in inputs if i["key"].lower().find("answer") < 0 and i["key"].lower().find("begin") < 0])
         return list(cpnts)
 
+    def get_input_elements(self):
+        key_set = set([])
+        res = []
+        for r in re.finditer(r"\{([a-z]+[:@][a-z0-9_-]+)\}", self._param.content, flags=re.IGNORECASE):
+            cpn_id = r.group(1)
+            if cpn_id in key_set:
+                continue
+            if cpn_id.lower().find("begin@") == 0:
+                cpn_id, key = cpn_id.split("@")
+                for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
+                    if p["key"] != key:
+                        continue
+                    res.append({"key": r.group(1), "name": p["name"]})
+                    key_set.add(r.group(1))
+                continue
+            cpn_nm = self._canvas.get_component_name(cpn_id)
+            if not cpn_nm:
+                continue
+            res.append({"key": cpn_id, "name": cpn_nm})
+            key_set.add(cpn_id)
+        return res
+
     def _run(self, history, **kwargs):
         content = self._param.content
 
         self._param.inputs = []
-        for para in self._param.parameters:
-            if not para.get("component_id"): continue
-            component_id = para["component_id"].split("@")[0]
-            if para["component_id"].lower().find("@") >= 0:
-                cpn_id, key = para["component_id"].split("@")
+        for para in self.get_input_elements():
+            if para["key"].lower().find("begin@") == 0:
+                cpn_id, key = para["key"].split("@")
                 for p in self._canvas.get_component(cpn_id)["obj"]._param.query:
                     if p["key"] == key:
-                        kwargs[para["key"]] = p.get("value", "")
-                        self._param.inputs.append(
-                            {"component_id": para["component_id"], "content": kwargs[para["key"]]})
+                        value = p.get("value", "")
+                        self.make_kwargs(para, kwargs, value)
                         break
                 else:
                     assert False, f"Can't find parameter '{key}' for {cpn_id}"
                 continue
 
+            component_id = para["key"]
             cpn = self._canvas.get_component(component_id)["obj"]
             if cpn.component_name.lower() == "answer":
                 hist = self._canvas.get_history(1)
@@ -68,18 +88,47 @@ class Template(ComponentBase):
                     hist = hist[0]["content"]
                 else:
                     hist = ""
-                kwargs[para["key"]] = hist
+                self.make_kwargs(para, kwargs, hist)
                 continue
 
             _, out = cpn.output(allow_partial=False)
-            if "content" not in out.columns:
-                kwargs[para["key"]] = ""
-            else:
-                kwargs[para["key"]] = "  - "+"\n - ".join([o if isinstance(o, str) else str(o) for o in out["content"]])
-            self._param.inputs.append({"component_id": para["component_id"], "content": kwargs[para["key"]]})
+
+            result = ""
+            if "content" in out.columns:
+                result = "\n".join(
+                    [o if isinstance(o, str) else str(o) for o in out["content"]]
+                )
+
+            self.make_kwargs(para, kwargs, result)
+
+        template = Jinja2Template(content)
+
+        try:
+            content = template.render(kwargs)
+        except Exception:
+            pass
 
         for n, v in kwargs.items():
-            content = re.sub(r"\{%s\}" % re.escape(n), str(v).replace("\\", " "), content)
+            if not isinstance(v, str):
+                try:
+                    v = json.dumps(v, ensure_ascii=False)
+                except Exception:
+                    pass
+            content = re.sub(
+                r"\{%s\}" % re.escape(n), v, content
+            )
+            content = re.sub(
+                r"(#+)", r" \1 ", content
+            )
 
         return Template.be_output(content)
 
+    def make_kwargs(self, para, kwargs, value):
+        self._param.inputs.append(
+            {"component_id": para["key"], "content": value}
+        )
+        try:
+            value = json.loads(value)
+        except Exception:
+            pass
+        kwargs[para["key"]] = value

agent/component/tushare.py

@@ -1,72 +1,72 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-import json
-from abc import ABC
-import pandas as pd
-import time
-import requests
-from agent.component.base import ComponentBase, ComponentParamBase
-
-
-class TuShareParam(ComponentParamBase):
-    """
-    Define the TuShare component parameters.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.token = "xxx"
-        self.src = "eastmoney"
-        self.start_date = "2024-01-01 09:00:00"
-        self.end_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
-        self.keyword = ""
-
-    def check(self):
-        self.check_valid_value(self.src, "Quick News Source",
-                               ["sina", "wallstreetcn", "10jqka", "eastmoney", "yuncaijing", "fenghuang", "jinrongjie"])
-
-
-class TuShare(ComponentBase, ABC):
-    component_name = "TuShare"
-
-    def _run(self, history, **kwargs):
-        ans = self.get_input()
-        ans = ",".join(ans["content"]) if "content" in ans else ""
-        if not ans:
-            return TuShare.be_output("")
-
-        try:
-            tus_res = []
-            params = {
-                "api_name": "news",
-                "token": self._param.token,
-                "params": {"src": self._param.src, "start_date": self._param.start_date,
-                           "end_date": self._param.end_date}
-            }
-            response = requests.post(url="http://api.tushare.pro", data=json.dumps(params).encode('utf-8'))
-            response = response.json()
-            if response['code'] != 0:
-                return TuShare.be_output(response['msg'])
-            df = pd.DataFrame(response['data']['items'])
-            df.columns = response['data']['fields']
-            tus_res.append({"content": (df[df['content'].str.contains(self._param.keyword, case=False)]).to_markdown()})
-        except Exception as e:
-            return TuShare.be_output("**ERROR**: " + str(e))
-
-        if not tus_res:
-            return TuShare.be_output("")
-
-        return pd.DataFrame(tus_res)
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import json
+from abc import ABC
+import pandas as pd
+import time
+import requests
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class TuShareParam(ComponentParamBase):
+    """
+    Define the TuShare component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.token = "xxx"
+        self.src = "eastmoney"
+        self.start_date = "2024-01-01 09:00:00"
+        self.end_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        self.keyword = ""
+
+    def check(self):
+        self.check_valid_value(self.src, "Quick News Source",
+                               ["sina", "wallstreetcn", "10jqka", "eastmoney", "yuncaijing", "fenghuang", "jinrongjie"])
+
+
+class TuShare(ComponentBase, ABC):
+    component_name = "TuShare"
+
+    def _run(self, history, **kwargs):
+        ans = self.get_input()
+        ans = ",".join(ans["content"]) if "content" in ans else ""
+        if not ans:
+            return TuShare.be_output("")
+
+        try:
+            tus_res = []
+            params = {
+                "api_name": "news",
+                "token": self._param.token,
+                "params": {"src": self._param.src, "start_date": self._param.start_date,
+                           "end_date": self._param.end_date}
+            }
+            response = requests.post(url="http://api.tushare.pro", data=json.dumps(params).encode('utf-8'))
+            response = response.json()
+            if response['code'] != 0:
+                return TuShare.be_output(response['msg'])
+            df = pd.DataFrame(response['data']['items'])
+            df.columns = response['data']['fields']
+            tus_res.append({"content": (df[df['content'].str.contains(self._param.keyword, case=False)]).to_markdown()})
+        except Exception as e:
+            return TuShare.be_output("**ERROR**: " + str(e))
+
+        if not tus_res:
+            return TuShare.be_output("")
+
+        return pd.DataFrame(tus_res)

agent/component/wencai.py

@@ -1,80 +1,80 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-from abc import ABC
-import pandas as pd
-import pywencai
-from agent.component.base import ComponentBase, ComponentParamBase
-
-
-class WenCaiParam(ComponentParamBase):
-    """
-    Define the WenCai component parameters.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.top_n = 10
-        self.query_type = "stock"
-
-    def check(self):
-        self.check_positive_integer(self.top_n, "Top N")
-        self.check_valid_value(self.query_type, "Query type",
-                               ['stock', 'zhishu', 'fund', 'hkstock', 'usstock', 'threeboard', 'conbond', 'insurance',
-                                'futures', 'lccp',
-                                'foreign_exchange'])
-
-
-class WenCai(ComponentBase, ABC):
-    component_name = "WenCai"
-
-    def _run(self, history, **kwargs):
-        ans = self.get_input()
-        ans = ",".join(ans["content"]) if "content" in ans else ""
-        if not ans:
-            return WenCai.be_output("")
-
-        try:
-            wencai_res = []
-            res = pywencai.get(query=ans, query_type=self._param.query_type, perpage=self._param.top_n)
-            if isinstance(res, pd.DataFrame):
-                wencai_res.append({"content": res.to_markdown()})
-            if isinstance(res, dict):
-                for item in res.items():
-                    if isinstance(item[1], list):
-                        wencai_res.append({"content": item[0] + "\n" + pd.DataFrame(item[1]).to_markdown()})
-                        continue
-                    if isinstance(item[1], str):
-                        wencai_res.append({"content": item[0] + "\n" + item[1]})
-                        continue
-                    if isinstance(item[1], dict):
-                        if "meta" in item[1].keys():
-                            continue
-                        wencai_res.append({"content": pd.DataFrame.from_dict(item[1], orient='index').to_markdown()})
-                        continue
-                    if isinstance(item[1], pd.DataFrame):
-                        if "image_url" in item[1].columns:
-                            continue
-                        wencai_res.append({"content": item[1].to_markdown()})
-                        continue
-                        
-                    wencai_res.append({"content": item[0] + "\n" + str(item[1])})
-        except Exception as e:
-            return WenCai.be_output("**ERROR**: " + str(e))
-
-        if not wencai_res:
-            return WenCai.be_output("")
-
-        return pd.DataFrame(wencai_res)
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from abc import ABC
+import pandas as pd
+import pywencai
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class WenCaiParam(ComponentParamBase):
+    """
+    Define the WenCai component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.top_n = 10
+        self.query_type = "stock"
+
+    def check(self):
+        self.check_positive_integer(self.top_n, "Top N")
+        self.check_valid_value(self.query_type, "Query type",
+                               ['stock', 'zhishu', 'fund', 'hkstock', 'usstock', 'threeboard', 'conbond', 'insurance',
+                                'futures', 'lccp',
+                                'foreign_exchange'])
+
+
+class WenCai(ComponentBase, ABC):
+    component_name = "WenCai"
+
+    def _run(self, history, **kwargs):
+        ans = self.get_input()
+        ans = ",".join(ans["content"]) if "content" in ans else ""
+        if not ans:
+            return WenCai.be_output("")
+
+        try:
+            wencai_res = []
+            res = pywencai.get(query=ans, query_type=self._param.query_type, perpage=self._param.top_n)
+            if isinstance(res, pd.DataFrame):
+                wencai_res.append({"content": res.to_markdown()})
+            if isinstance(res, dict):
+                for item in res.items():
+                    if isinstance(item[1], list):
+                        wencai_res.append({"content": item[0] + "\n" + pd.DataFrame(item[1]).to_markdown()})
+                        continue
+                    if isinstance(item[1], str):
+                        wencai_res.append({"content": item[0] + "\n" + item[1]})
+                        continue
+                    if isinstance(item[1], dict):
+                        if "meta" in item[1].keys():
+                            continue
+                        wencai_res.append({"content": pd.DataFrame.from_dict(item[1], orient='index').to_markdown()})
+                        continue
+                    if isinstance(item[1], pd.DataFrame):
+                        if "image_url" in item[1].columns:
+                            continue
+                        wencai_res.append({"content": item[1].to_markdown()})
+                        continue
+                        
+                    wencai_res.append({"content": item[0] + "\n" + str(item[1])})
+        except Exception as e:
+            return WenCai.be_output("**ERROR**: " + str(e))
+
+        if not wencai_res:
+            return WenCai.be_output("")
+
+        return pd.DataFrame(wencai_res)

agent/component/yahoofinance.py

@@ -1,84 +1,84 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-import logging
-from abc import ABC
-import pandas as pd
-from agent.component.base import ComponentBase, ComponentParamBase
-import yfinance as yf
-
-
-class YahooFinanceParam(ComponentParamBase):
-    """
-    Define the YahooFinance component parameters.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.info = True
-        self.history = False
-        self.count = False
-        self.financials = False
-        self.income_stmt = False
-        self.balance_sheet = False
-        self.cash_flow_statement = False
-        self.news = True
-
-    def check(self):
-        self.check_boolean(self.info, "get all stock info")
-        self.check_boolean(self.history, "get historical market data")
-        self.check_boolean(self.count, "show share count")
-        self.check_boolean(self.financials, "show financials")
-        self.check_boolean(self.income_stmt, "income statement")
-        self.check_boolean(self.balance_sheet, "balance sheet")
-        self.check_boolean(self.cash_flow_statement, "cash flow statement")
-        self.check_boolean(self.news, "show news")
-
-
-class YahooFinance(ComponentBase, ABC):
-    component_name = "YahooFinance"
-
-    def _run(self, history, **kwargs):
-        ans = self.get_input()
-        ans = "".join(ans["content"]) if "content" in ans else ""
-        if not ans:
-            return YahooFinance.be_output("")
-
-        yohoo_res = []
-        try:
-            msft = yf.Ticker(ans)
-            if self._param.info:
-                yohoo_res.append({"content": "info:\n" + pd.Series(msft.info).to_markdown() + "\n"})
-            if self._param.history:
-                yohoo_res.append({"content": "history:\n" + msft.history().to_markdown() + "\n"})
-            if self._param.financials:
-                yohoo_res.append({"content": "calendar:\n" + pd.DataFrame(msft.calendar).to_markdown() + "\n"})
-            if self._param.balance_sheet:
-                yohoo_res.append({"content": "balance sheet:\n" + msft.balance_sheet.to_markdown() + "\n"})
-                yohoo_res.append(
-                    {"content": "quarterly balance sheet:\n" + msft.quarterly_balance_sheet.to_markdown() + "\n"})
-            if self._param.cash_flow_statement:
-                yohoo_res.append({"content": "cash flow statement:\n" + msft.cashflow.to_markdown() + "\n"})
-                yohoo_res.append(
-                    {"content": "quarterly cash flow statement:\n" + msft.quarterly_cashflow.to_markdown() + "\n"})
-            if self._param.news:
-                yohoo_res.append({"content": "news:\n" + pd.DataFrame(msft.news).to_markdown() + "\n"})
-        except Exception:
-            logging.exception("YahooFinance got exception")
-
-        if not yohoo_res:
-            return YahooFinance.be_output("")
-
-        return pd.DataFrame(yohoo_res)
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import logging
+from abc import ABC
+import pandas as pd
+from agent.component.base import ComponentBase, ComponentParamBase
+import yfinance as yf
+
+
+class YahooFinanceParam(ComponentParamBase):
+    """
+    Define the YahooFinance component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.info = True
+        self.history = False
+        self.count = False
+        self.financials = False
+        self.income_stmt = False
+        self.balance_sheet = False
+        self.cash_flow_statement = False
+        self.news = True
+
+    def check(self):
+        self.check_boolean(self.info, "get all stock info")
+        self.check_boolean(self.history, "get historical market data")
+        self.check_boolean(self.count, "show share count")
+        self.check_boolean(self.financials, "show financials")
+        self.check_boolean(self.income_stmt, "income statement")
+        self.check_boolean(self.balance_sheet, "balance sheet")
+        self.check_boolean(self.cash_flow_statement, "cash flow statement")
+        self.check_boolean(self.news, "show news")
+
+
+class YahooFinance(ComponentBase, ABC):
+    component_name = "YahooFinance"
+
+    def _run(self, history, **kwargs):
+        ans = self.get_input()
+        ans = "".join(ans["content"]) if "content" in ans else ""
+        if not ans:
+            return YahooFinance.be_output("")
+
+        yohoo_res = []
+        try:
+            msft = yf.Ticker(ans)
+            if self._param.info:
+                yohoo_res.append({"content": "info:\n" + pd.Series(msft.info).to_markdown() + "\n"})
+            if self._param.history:
+                yohoo_res.append({"content": "history:\n" + msft.history().to_markdown() + "\n"})
+            if self._param.financials:
+                yohoo_res.append({"content": "calendar:\n" + pd.DataFrame(msft.calendar).to_markdown() + "\n"})
+            if self._param.balance_sheet:
+                yohoo_res.append({"content": "balance sheet:\n" + msft.balance_sheet.to_markdown() + "\n"})
+                yohoo_res.append(
+                    {"content": "quarterly balance sheet:\n" + msft.quarterly_balance_sheet.to_markdown() + "\n"})
+            if self._param.cash_flow_statement:
+                yohoo_res.append({"content": "cash flow statement:\n" + msft.cashflow.to_markdown() + "\n"})
+                yohoo_res.append(
+                    {"content": "quarterly cash flow statement:\n" + msft.quarterly_cashflow.to_markdown() + "\n"})
+            if self._param.news:
+                yohoo_res.append({"content": "news:\n" + pd.DataFrame(msft.news).to_markdown() + "\n"})
+        except Exception:
+            logging.exception("YahooFinance got exception")
+
+        if not yohoo_res:
+            return YahooFinance.be_output("")
+
+        return pd.DataFrame(yohoo_res)

agent/settings.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

agent/test/client.py

@@ -43,6 +43,7 @@ if __name__ == '__main__':
         else:
             print(ans["content"])
 
-        if DEBUG: print(canvas.path)
+        if DEBUG:
+            print(canvas.path)
         question = input("\n==================== User =====================\n> ")
         canvas.add_user_input(question)

agent/test/dsl_examples/concentrator_message.json

@@ -1,113 +1,113 @@
-{
-  "components": {
-            "begin": {
-                "obj":{
-                    "component_name": "Begin",
-                    "params": {
-                      "prologue": "Hi there!"
-                    }
-                },
-                "downstream": ["answer:0"],
-                "upstream": []
-            },
-            "answer:0": {
-                "obj": {
-                    "component_name": "Answer",
-                    "params": {}
-                },
-                "downstream": ["categorize:0"],
-                "upstream": ["begin"]
-            },
-            "categorize:0": {
-                "obj": {
-                    "component_name": "Categorize",
-                    "params": {
-                      "llm_id": "deepseek-chat",
-                      "category_description": {
-                        "product_related": {
-                          "description": "The question is about the product usage, appearance and how it works.",
-                          "examples": "Why it always beaming?\nHow to install it onto the wall?\nIt leaks, what to do?",
-                          "to": "concentrator:0"
-                        },
-                        "others": {
-                          "description": "The question is not about the product usage, appearance and how it works.",
-                          "examples": "How are you doing?\nWhat is your name?\nAre you a robot?\nWhat's the weather?\nWill it rain?",
-                          "to": "concentrator:1"
-                        }
-                      }
-                    }
-                },
-                "downstream": ["concentrator:0","concentrator:1"],
-                "upstream": ["answer:0"]
-            },
-            "concentrator:0": {
-                "obj": {
-                    "component_name": "Concentrator",
-                    "params": {}
-                },
-                "downstream": ["message:0"],
-                "upstream": ["categorize:0"]
-            },
-            "concentrator:1": {
-                "obj": {
-                    "component_name": "Concentrator",
-                    "params": {}
-                },
-                "downstream": ["message:1_0","message:1_1","message:1_2"],
-                "upstream": ["categorize:0"]
-            },
-            "message:0": {
-                "obj": {
-                    "component_name": "Message",
-                    "params": {
-                      "messages": [
-                        "Message 0_0!!!!!!!"
-                      ]
-                    }
-                },
-                "downstream": ["answer:0"],
-                "upstream": ["concentrator:0"]
-            },
-            "message:1_0": {
-                "obj": {
-                    "component_name": "Message",
-                    "params": {
-                      "messages": [
-                        "Message 1_0!!!!!!!"
-                      ]
-                    }
-                },
-                "downstream": ["answer:0"],
-                "upstream": ["concentrator:1"]
-            },
-            "message:1_1": {
-                "obj": {
-                    "component_name": "Message",
-                    "params": {
-                      "messages": [
-                        "Message 1_1!!!!!!!"
-                      ]
-                    }
-                },
-                "downstream": ["answer:0"],
-                "upstream": ["concentrator:1"]
-            },
-            "message:1_2": {
-                "obj": {
-                    "component_name": "Message",
-                    "params": {
-                      "messages": [
-                        "Message 1_2!!!!!!!"
-                      ]
-                    }
-                },
-                "downstream": ["answer:0"],
-                "upstream": ["concentrator:1"]
-            }
-  },
-  "history": [],
-  "messages": [],
-  "path": [],
-  "reference": [],
-  "answer": []
+{
+  "components": {
+            "begin": {
+                "obj":{
+                    "component_name": "Begin",
+                    "params": {
+                      "prologue": "Hi there!"
+                    }
+                },
+                "downstream": ["answer:0"],
+                "upstream": []
+            },
+            "answer:0": {
+                "obj": {
+                    "component_name": "Answer",
+                    "params": {}
+                },
+                "downstream": ["categorize:0"],
+                "upstream": ["begin"]
+            },
+            "categorize:0": {
+                "obj": {
+                    "component_name": "Categorize",
+                    "params": {
+                      "llm_id": "deepseek-chat",
+                      "category_description": {
+                        "product_related": {
+                          "description": "The question is about the product usage, appearance and how it works.",
+                          "examples": "Why it always beaming?\nHow to install it onto the wall?\nIt leaks, what to do?",
+                          "to": "concentrator:0"
+                        },
+                        "others": {
+                          "description": "The question is not about the product usage, appearance and how it works.",
+                          "examples": "How are you doing?\nWhat is your name?\nAre you a robot?\nWhat's the weather?\nWill it rain?",
+                          "to": "concentrator:1"
+                        }
+                      }
+                    }
+                },
+                "downstream": ["concentrator:0","concentrator:1"],
+                "upstream": ["answer:0"]
+            },
+            "concentrator:0": {
+                "obj": {
+                    "component_name": "Concentrator",
+                    "params": {}
+                },
+                "downstream": ["message:0"],
+                "upstream": ["categorize:0"]
+            },
+            "concentrator:1": {
+                "obj": {
+                    "component_name": "Concentrator",
+                    "params": {}
+                },
+                "downstream": ["message:1_0","message:1_1","message:1_2"],
+                "upstream": ["categorize:0"]
+            },
+            "message:0": {
+                "obj": {
+                    "component_name": "Message",
+                    "params": {
+                      "messages": [
+                        "Message 0_0!!!!!!!"
+                      ]
+                    }
+                },
+                "downstream": ["answer:0"],
+                "upstream": ["concentrator:0"]
+            },
+            "message:1_0": {
+                "obj": {
+                    "component_name": "Message",
+                    "params": {
+                      "messages": [
+                        "Message 1_0!!!!!!!"
+                      ]
+                    }
+                },
+                "downstream": ["answer:0"],
+                "upstream": ["concentrator:1"]
+            },
+            "message:1_1": {
+                "obj": {
+                    "component_name": "Message",
+                    "params": {
+                      "messages": [
+                        "Message 1_1!!!!!!!"
+                      ]
+                    }
+                },
+                "downstream": ["answer:0"],
+                "upstream": ["concentrator:1"]
+            },
+            "message:1_2": {
+                "obj": {
+                    "component_name": "Message",
+                    "params": {
+                      "messages": [
+                        "Message 1_2!!!!!!!"
+                      ]
+                    }
+                },
+                "downstream": ["answer:0"],
+                "upstream": ["concentrator:1"]
+            }
+  },
+  "history": [],
+  "messages": [],
+  "path": [],
+  "reference": [],
+  "answer": []
 }

agentic_reasoning/__init__.py

@@ -0,0 +1 @@
+from .deep_research import DeepResearcher as DeepResearcher

agentic_reasoning/deep_research.py

@@ -0,0 +1,223 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import logging
+import re
+from functools import partial
+from agentic_reasoning.prompts import BEGIN_SEARCH_QUERY, BEGIN_SEARCH_RESULT, END_SEARCH_RESULT, MAX_SEARCH_LIMIT, \
+    END_SEARCH_QUERY, REASON_PROMPT, RELEVANT_EXTRACTION_PROMPT
+from api.db.services.llm_service import LLMBundle
+from rag.nlp import extract_between
+from rag.prompts import kb_prompt
+from rag.utils.tavily_conn import Tavily
+
+
+class DeepResearcher:
+    def __init__(self,
+                 chat_mdl: LLMBundle,
+                 prompt_config: dict,
+                 kb_retrieve: partial = None,
+                 kg_retrieve: partial = None
+                 ):
+        self.chat_mdl = chat_mdl
+        self.prompt_config = prompt_config
+        self._kb_retrieve = kb_retrieve
+        self._kg_retrieve = kg_retrieve
+
+    @staticmethod
+    def _remove_query_tags(text):
+        """Remove query tags from text"""
+        pattern = re.escape(BEGIN_SEARCH_QUERY) + r"(.*?)" + re.escape(END_SEARCH_QUERY)
+        return re.sub(pattern, "", text)
+
+    @staticmethod
+    def _remove_result_tags(text):
+        """Remove result tags from text"""
+        pattern = re.escape(BEGIN_SEARCH_RESULT) + r"(.*?)" + re.escape(END_SEARCH_RESULT)
+        return re.sub(pattern, "", text)
+
+    def _generate_reasoning(self, msg_history):
+        """Generate reasoning steps"""
+        query_think = ""
+        if msg_history[-1]["role"] != "user":
+            msg_history.append({"role": "user", "content": "Continues reasoning with the new information.\n"})
+        else:
+            msg_history[-1]["content"] += "\n\nContinues reasoning with the new information.\n"
+            
+        for ans in self.chat_mdl.chat_streamly(REASON_PROMPT, msg_history, {"temperature": 0.7}):
+            ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+            if not ans:
+                continue
+            query_think = ans
+            yield query_think
+        return query_think
+
+    def _extract_search_queries(self, query_think, question, step_index):
+        """Extract search queries from thinking"""
+        queries = extract_between(query_think, BEGIN_SEARCH_QUERY, END_SEARCH_QUERY)
+        if not queries and step_index == 0:
+            # If this is the first step and no queries are found, use the original question as the query
+            queries = [question]
+        return queries
+
+    def _truncate_previous_reasoning(self, all_reasoning_steps):
+        """Truncate previous reasoning steps to maintain a reasonable length"""
+        truncated_prev_reasoning = ""
+        for i, step in enumerate(all_reasoning_steps):
+            truncated_prev_reasoning += f"Step {i + 1}: {step}\n\n"
+
+        prev_steps = truncated_prev_reasoning.split('\n\n')
+        if len(prev_steps) <= 5:
+            truncated_prev_reasoning = '\n\n'.join(prev_steps)
+        else:
+            truncated_prev_reasoning = ''
+            for i, step in enumerate(prev_steps):
+                if i == 0 or i >= len(prev_steps) - 4 or BEGIN_SEARCH_QUERY in step or BEGIN_SEARCH_RESULT in step:
+                    truncated_prev_reasoning += step + '\n\n'
+                else:
+                    if truncated_prev_reasoning[-len('\n\n...\n\n'):] != '\n\n...\n\n':
+                        truncated_prev_reasoning += '...\n\n'
+        
+        return truncated_prev_reasoning.strip('\n')
+
+    def _retrieve_information(self, search_query):
+        """Retrieve information from different sources"""
+        # 1. Knowledge base retrieval
+        kbinfos = self._kb_retrieve(question=search_query) if self._kb_retrieve else {"chunks": [], "doc_aggs": []}
+        
+        # 2. Web retrieval (if Tavily API is configured)
+        if self.prompt_config.get("tavily_api_key"):
+            tav = Tavily(self.prompt_config["tavily_api_key"])
+            tav_res = tav.retrieve_chunks(search_query)
+            kbinfos["chunks"].extend(tav_res["chunks"])
+            kbinfos["doc_aggs"].extend(tav_res["doc_aggs"])
+        
+        # 3. Knowledge graph retrieval (if configured)
+        if self.prompt_config.get("use_kg") and self._kg_retrieve:
+            ck = self._kg_retrieve(question=search_query)
+            if ck["content_with_weight"]:
+                kbinfos["chunks"].insert(0, ck)
+                
+        return kbinfos
+
+    def _update_chunk_info(self, chunk_info, kbinfos):
+        """Update chunk information for citations"""
+        if not chunk_info["chunks"]:
+            # If this is the first retrieval, use the retrieval results directly
+            for k in chunk_info.keys():
+                chunk_info[k] = kbinfos[k]
+        else:
+            # Merge newly retrieved information, avoiding duplicates
+            cids = [c["chunk_id"] for c in chunk_info["chunks"]]
+            for c in kbinfos["chunks"]:
+                if c["chunk_id"] not in cids:
+                    chunk_info["chunks"].append(c)
+                    
+            dids = [d["doc_id"] for d in chunk_info["doc_aggs"]]
+            for d in kbinfos["doc_aggs"]:
+                if d["doc_id"] not in dids:
+                    chunk_info["doc_aggs"].append(d)
+
+    def _extract_relevant_info(self, truncated_prev_reasoning, search_query, kbinfos):
+        """Extract and summarize relevant information"""
+        summary_think = ""
+        for ans in self.chat_mdl.chat_streamly(
+                RELEVANT_EXTRACTION_PROMPT.format(
+                    prev_reasoning=truncated_prev_reasoning,
+                    search_query=search_query,
+                    document="\n".join(kb_prompt(kbinfos, 4096))
+                ),
+                [{"role": "user",
+                  "content": f'Now you should analyze each web page and find helpful information based on the current search query "{search_query}" and previous reasoning steps.'}],
+                {"temperature": 0.7}):
+            ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+            if not ans:
+                continue
+            summary_think = ans
+            yield summary_think
+        
+        return summary_think
+
+    def thinking(self, chunk_info: dict, question: str):
+        executed_search_queries = []
+        msg_history = [{"role": "user", "content": f'Question:\"{question}\"\n'}]
+        all_reasoning_steps = []
+        think = "<think>"
+        
+        for step_index in range(MAX_SEARCH_LIMIT + 1):
+            # Check if the maximum search limit has been reached
+            if step_index == MAX_SEARCH_LIMIT - 1:
+                summary_think = f"\n{BEGIN_SEARCH_RESULT}\nThe maximum search limit is exceeded. You are not allowed to search.\n{END_SEARCH_RESULT}\n"
+                yield {"answer": think + summary_think + "</think>", "reference": {}, "audio_binary": None}
+                all_reasoning_steps.append(summary_think)
+                msg_history.append({"role": "assistant", "content": summary_think})
+                break
+
+            # Step 1: Generate reasoning
+            query_think = ""
+            for ans in self._generate_reasoning(msg_history):
+                query_think = ans
+                yield {"answer": think + self._remove_query_tags(query_think) + "</think>", "reference": {}, "audio_binary": None}
+
+            think += self._remove_query_tags(query_think)
+            all_reasoning_steps.append(query_think)
+            
+            # Step 2: Extract search queries
+            queries = self._extract_search_queries(query_think, question, step_index)
+            if not queries and step_index > 0:
+                # If not the first step and no queries, end the search process
+                break
+
+            # Process each search query
+            for search_query in queries:
+                logging.info(f"[THINK]Query: {step_index}. {search_query}")
+                msg_history.append({"role": "assistant", "content": search_query})
+                think += f"\n\n> {step_index + 1}. {search_query}\n\n"
+                yield {"answer": think + "</think>", "reference": {}, "audio_binary": None}
+
+                # Check if the query has already been executed
+                if search_query in executed_search_queries:
+                    summary_think = f"\n{BEGIN_SEARCH_RESULT}\nYou have searched this query. Please refer to previous results.\n{END_SEARCH_RESULT}\n"
+                    yield {"answer": think + summary_think + "</think>", "reference": {}, "audio_binary": None}
+                    all_reasoning_steps.append(summary_think)
+                    msg_history.append({"role": "user", "content": summary_think})
+                    think += summary_think
+                    continue
+                
+                executed_search_queries.append(search_query)
+                
+                # Step 3: Truncate previous reasoning steps
+                truncated_prev_reasoning = self._truncate_previous_reasoning(all_reasoning_steps)
+                
+                # Step 4: Retrieve information
+                kbinfos = self._retrieve_information(search_query)
+                
+                # Step 5: Update chunk information
+                self._update_chunk_info(chunk_info, kbinfos)
+                
+                # Step 6: Extract relevant information
+                think += "\n\n"
+                summary_think = ""
+                for ans in self._extract_relevant_info(truncated_prev_reasoning, search_query, kbinfos):
+                    summary_think = ans
+                    yield {"answer": think + self._remove_result_tags(summary_think) + "</think>", "reference": {}, "audio_binary": None}
+
+                all_reasoning_steps.append(summary_think)
+                msg_history.append(
+                    {"role": "user", "content": f"\n\n{BEGIN_SEARCH_RESULT}{summary_think}{END_SEARCH_RESULT}\n\n"})
+                think += self._remove_result_tags(summary_think)
+                logging.info(f"[THINK]Summary: {step_index}. {summary_think}")
+
+        yield think + "</think>"

agentic_reasoning/prompts.py

@@ -0,0 +1,113 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+BEGIN_SEARCH_QUERY = "<|begin_search_query|>"
+END_SEARCH_QUERY = "<|end_search_query|>"
+BEGIN_SEARCH_RESULT = "<|begin_search_result|>"
+END_SEARCH_RESULT = "<|end_search_result|>"
+MAX_SEARCH_LIMIT = 6
+
+REASON_PROMPT = (
+        "You are a reasoning assistant with the ability to perform dataset searches to help "
+        "you answer the user's question accurately. You have special tools:\n\n"
+        f"- To perform a search: write {BEGIN_SEARCH_QUERY} your query here {END_SEARCH_QUERY}.\n"
+        f"Then, the system will search and analyze relevant content, then provide you with helpful information in the format {BEGIN_SEARCH_RESULT} ...search results... {END_SEARCH_RESULT}.\n\n"
+        f"You can repeat the search process multiple times if necessary. The maximum number of search attempts is limited to {MAX_SEARCH_LIMIT}.\n\n"
+        "Once you have all the information you need, continue your reasoning.\n\n"
+        "-- Example 1 --\n" ########################################
+        "Question: \"Are both the directors of Jaws and Casino Royale from the same country?\"\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Who is the director of Jaws?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nThe director of Jaws is Steven Spielberg...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information.\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Where is Steven Spielberg from?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nSteven Allan Spielberg is an American filmmaker...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information...\n\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Who is the director of Casino Royale?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nCasino Royale is a 2006 spy film directed by Martin Campbell...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information...\n\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Where is Martin Campbell from?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nMartin Campbell (born 24 October 1943) is a New Zealand film and television director...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information...\n\n"
+        "Assistant:\nIt's enough to answer the question\n"
+
+        "-- Example 2 --\n" #########################################
+        "Question: \"When was the founder of craigslist born?\"\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY}Who was the founder of craigslist?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nCraigslist was founded by Craig Newmark...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information.\n"
+        "Assistant:\n"
+        f"    {BEGIN_SEARCH_QUERY} When was Craig Newmark born?{END_SEARCH_QUERY}\n\n"
+        "User:\n"
+        f"    {BEGIN_SEARCH_RESULT}\nCraig Newmark was born on December 6, 1952...\n{END_SEARCH_RESULT}\n\n"
+        "Continues reasoning with the new information...\n\n"
+        "Assistant:\nIt's enough to answer the question\n"
+        "**Remember**:\n"
+        f"- You have a dataset to search, so you just provide a proper search query.\n"
+        f"- Use {BEGIN_SEARCH_QUERY} to request a dataset search and end with {END_SEARCH_QUERY}.\n"
+        "- The language of query MUST be as the same as 'Question' or 'search result'.\n"
+        "- If no helpful information can be found, rewrite the search query to be less and precise keywords.\n"
+        "- When done searching, continue your reasoning.\n\n"
+        'Please answer the following question. You should think step by step to solve it.\n\n'
+    )
+
+RELEVANT_EXTRACTION_PROMPT = """**Task Instruction:**
+
+    You are tasked with reading and analyzing web pages based on the following inputs: **Previous Reasoning Steps**, **Current Search Query**, and **Searched Web Pages**. Your objective is to extract relevant and helpful information for **Current Search Query** from the **Searched Web Pages** and seamlessly integrate this information into the **Previous Reasoning Steps** to continue reasoning for the original question.
+
+    **Guidelines:**
+
+    1. **Analyze the Searched Web Pages:**
+    - Carefully review the content of each searched web page.
+    - Identify factual information that is relevant to the **Current Search Query** and can aid in the reasoning process for the original question.
+
+    2. **Extract Relevant Information:**
+    - Select the information from the Searched Web Pages that directly contributes to advancing the **Previous Reasoning Steps**.
+    - Ensure that the extracted information is accurate and relevant.
+
+    3. **Output Format:**
+    - **If the web pages provide helpful information for current search query:** Present the information beginning with `**Final Information**` as shown below.
+    - The language of query **MUST BE** as the same as 'Search Query' or 'Web Pages'.\n"
+    **Final Information**
+
+    [Helpful information]
+
+    - **If the web pages do not provide any helpful information for current search query:** Output the following text.
+
+    **Final Information**
+
+    No helpful information found.
+
+    **Inputs:**
+    - **Previous Reasoning Steps:**  
+    {prev_reasoning}
+
+    - **Current Search Query:**  
+    {search_query}
+
+    - **Searched Web Pages:**  
+    {document}
+
+    """

api/__init__.py

@@ -0,0 +1,18 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from beartype.claw import beartype_this_package
+beartype_this_package()

api/apps/__init__.py

@@ -83,7 +83,7 @@ app.errorhandler(Exception)(server_error_response)
 app.config["SESSION_PERMANENT"] = False
 app.config["SESSION_TYPE"] = "filesystem"
 app.config["MAX_CONTENT_LENGTH"] = int(
-    os.environ.get("MAX_CONTENT_LENGTH", 128 * 1024 * 1024)
+    os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024 * 1024)
 )
 
 Session(app)
@@ -107,7 +107,7 @@ def search_pages_path(pages_dir):
 def register_page(page_path):
     path = f"{page_path}"
 
-    page_name = page_path.stem.rstrip("_app")
+    page_name = page_path.stem.removesuffix("_app")
     module_name = ".".join(
         page_path.parts[page_path.parts.index("api"): -1] + (page_name,)
     )
@@ -119,8 +119,9 @@ def register_page(page_path):
     sys.modules[module_name] = page
     spec.loader.exec_module(page)
     page_name = getattr(page, "page_name", page_name)
+    sdk_path = "\\sdk\\" if sys.platform.startswith("win") else "/sdk/"
     url_prefix = (
-        f"/api/{API_VERSION}" if "/sdk/" in path else f"/{API_VERSION}/{page_name}"
+        f"/api/{API_VERSION}" if sdk_path in path else f"/{API_VERSION}/{page_name}"
     )
 
     app.register_blueprint(page.manager, url_prefix=url_prefix)
@@ -152,8 +153,8 @@ def load_user(web_request):
                 return user[0]
             else:
                 return None
-        except Exception:
-            logging.exception("load_user got exception")
+        except Exception as e:
+            logging.warning(f"load_user got exception {e}")
             return None
     else:
         return None

api/apps/api_app.py

@@ -21,11 +21,11 @@ from flask import request, Response
 from api.db.services.llm_service import TenantLLMService
 from flask_login import login_required, current_user
 
-from api.db import FileType, LLMType, ParserType, FileSource
+from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileType, LLMType, ParserType, FileSource
 from api.db.db_models import APIToken, Task, File
 from api.db.services import duplicate_name
 from api.db.services.api_service import APITokenService, API4ConversationService
-from api.db.services.dialog_service import DialogService, chat, keyword_extraction
+from api.db.services.dialog_service import DialogService, chat
 from api.db.services.document_service import DocumentService, doc_upload_and_parse
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
@@ -38,6 +38,8 @@ from api.utils.api_utils import server_error_response, get_data_error_result, ge
     generate_confirmation_token
 
 from api.utils.file_utils import filename_type, thumbnail
+from rag.app.tag import label_question
+from rag.prompts import keyword_extraction
 from rag.utils.storage_factory import STORAGE_IMPL
 
 from api.db.services.canvas_service import UserCanvasService
@@ -45,7 +47,7 @@ from agent.canvas import Canvas
 from functools import partial
 
 
-@manager.route('/new_token', methods=['POST'])
+@manager.route('/new_token', methods=['POST'])  # noqa: F821
 @login_required
 def new_token():
     req = request.json
@@ -75,7 +77,7 @@ def new_token():
         return server_error_response(e)
 
 
-@manager.route('/token_list', methods=['GET'])
+@manager.route('/token_list', methods=['GET'])  # noqa: F821
 @login_required
 def token_list():
     try:
@@ -90,7 +92,7 @@ def token_list():
         return server_error_response(e)
 
 
-@manager.route('/rm', methods=['POST'])
+@manager.route('/rm', methods=['POST'])  # noqa: F821
 @validate_request("tokens", "tenant_id")
 @login_required
 def rm():
@@ -104,7 +106,7 @@ def rm():
         return server_error_response(e)
 
 
-@manager.route('/stats', methods=['GET'])
+@manager.route('/stats', methods=['GET'])  # noqa: F821
 @login_required
 def stats():
     try:
@@ -135,14 +137,13 @@ def stats():
         return server_error_response(e)
 
 
-@manager.route('/new_conversation', methods=['GET'])
+@manager.route('/new_conversation', methods=['GET'])  # noqa: F821
 def set_conversation():
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
-    req = request.json
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
     try:
         if objs[0].source == "agent":
             e, cvs = UserCanvasService.get_by_id(objs[0].dialog_id)
@@ -176,19 +177,20 @@ def set_conversation():
         return server_error_response(e)
 
 
-@manager.route('/completion', methods=['POST'])
+@manager.route('/completion', methods=['POST'])  # noqa: F821
 @validate_request("conversation_id", "messages")
 def completion():
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
     req = request.json
     e, conv = API4ConversationService.get_by_id(req["conversation_id"])
     if not e:
         return get_data_error_result(message="Conversation not found!")
-    if "quote" not in req: req["quote"] = False
+    if "quote" not in req:
+        req["quote"] = False
 
     msg = []
     for m in req["messages"]:
@@ -197,7 +199,8 @@ def completion():
         if m["role"] == "assistant" and not msg:
             continue
         msg.append(m)
-    if not msg[-1].get("id"): msg[-1]["id"] = get_uuid()
+    if not msg[-1].get("id"):
+        msg[-1]["id"] = get_uuid()
     message_id = msg[-1]["id"]
 
     def fillin_conv(ans):
@@ -340,14 +343,14 @@ def completion():
         return server_error_response(e)
 
 
-@manager.route('/conversation/<conversation_id>', methods=['GET'])
+@manager.route('/conversation/<conversation_id>', methods=['GET'])  # noqa: F821
 # @login_required
-def get(conversation_id):
+def get_conversation(conversation_id):
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
 
     try:
         e, conv = API4ConversationService.get_by_id(conversation_id)
@@ -356,7 +359,7 @@ def get(conversation_id):
 
         conv = conv.to_dict()
         if token != APIToken.query(dialog_id=conv['dialog_id'])[0].token:
-            return get_json_result(data=False, message='Token is not valid for this conversation_id!"',
+            return get_json_result(data=False, message='Authentication error: API key is invalid for this conversation_id!"',
                                    code=settings.RetCode.AUTHENTICATION_ERROR)
 
         for referenct_i in conv['reference']:
@@ -371,14 +374,14 @@ def get(conversation_id):
         return server_error_response(e)
 
 
-@manager.route('/document/upload', methods=['POST'])
+@manager.route('/document/upload', methods=['POST'])  # noqa: F821
 @validate_request("kb_name")
 def upload():
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
 
     kb_name = request.form.get("kb_name").strip()
     tenant_id = objs[0].tenant_id
@@ -476,21 +479,21 @@ def upload():
                 doc = doc.to_dict()
                 doc["tenant_id"] = tenant_id
                 bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
-                queue_tasks(doc, bucket, name)
+                queue_tasks(doc, bucket, name, 0)
             except Exception as e:
                 return server_error_response(e)
 
     return get_json_result(data=doc_result.to_json())
 
 
-@manager.route('/document/upload_and_parse', methods=['POST'])
+@manager.route('/document/upload_and_parse', methods=['POST'])  # noqa: F821
 @validate_request("conversation_id")
 def upload_parse():
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
 
     if 'file' not in request.files:
         return get_json_result(
@@ -506,14 +509,14 @@ def upload_parse():
     return get_json_result(data=doc_ids)
 
 
-@manager.route('/list_chunks', methods=['POST'])
+@manager.route('/list_chunks', methods=['POST'])  # noqa: F821
 # @login_required
 def list_chunks():
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
 
@@ -545,15 +548,40 @@ def list_chunks():
 
     return get_json_result(data=res)
 
+@manager.route('/get_chunk/<chunk_id>', methods=['GET'])  # noqa: F821
+# @login_required
+def get_chunk(chunk_id):
+    from rag.nlp import search
+    token = request.headers.get('Authorization').split()[1]
+    objs = APIToken.query(token=token)
+    if not objs:
+        return get_json_result(
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+    try:
+        tenant_id = objs[0].tenant_id
+        kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
+        chunk = settings.docStoreConn.get(chunk_id, search.index_name(tenant_id), kb_ids)
+        if chunk is None:
+            return server_error_response(Exception("Chunk not found"))
+        k = []
+        for n in chunk.keys():
+            if re.search(r"(_vec$|_sm_|_tks|_ltks)", n):
+                k.append(n)
+        for n in k:
+            del chunk[n]
 
-@manager.route('/list_kb_docs', methods=['POST'])
+        return get_json_result(data=chunk)
+    except Exception as e:
+        return server_error_response(e)
+
+@manager.route('/list_kb_docs', methods=['POST'])  # noqa: F821
 # @login_required
 def list_kb_docs():
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
     tenant_id = objs[0].tenant_id
@@ -574,10 +602,23 @@ def list_kb_docs():
     orderby = req.get("orderby", "create_time")
     desc = req.get("desc", True)
     keywords = req.get("keywords", "")
-
+    status = req.get("status", [])
+    if status:
+        invalid_status = {s for s in status if s not in VALID_TASK_STATUS}
+        if invalid_status:
+            return get_data_error_result(
+                message=f"Invalid filter status conditions: {', '.join(invalid_status)}"
+            )
+    types = req.get("types", [])
+    if types:
+        invalid_types = {t for t in types if t not in VALID_FILE_TYPES}
+        if invalid_types:
+            return get_data_error_result(
+                message=f"Invalid filter conditions: {', '.join(invalid_types)} type{'s' if len(invalid_types) > 1 else ''}"
+            )
     try:
         docs, tol = DocumentService.get_by_kb_id(
-            kb_id, page_number, items_per_page, orderby, desc, keywords)
+            kb_id, page_number, items_per_page, orderby, desc, keywords, status, types)
         docs = [{"doc_id": doc['id'], "doc_name": doc['name']} for doc in docs]
 
         return get_json_result(data={"total": tol, "docs": docs})
@@ -586,33 +627,33 @@ def list_kb_docs():
         return server_error_response(e)
 
 
-@manager.route('/document/infos', methods=['POST'])
+@manager.route('/document/infos', methods=['POST'])  # noqa: F821
 @validate_request("doc_ids")
 def docinfos():
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
     req = request.json
     doc_ids = req["doc_ids"]
     docs = DocumentService.get_by_ids(doc_ids)
     return get_json_result(data=list(docs.dicts()))
 
 
-@manager.route('/document', methods=['DELETE'])
+@manager.route('/document', methods=['DELETE'])  # noqa: F821
 # @login_required
 def document_rm():
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
 
     tenant_id = objs[0].tenant_id
     req = request.json
     try:
-        doc_ids = [DocumentService.get_doc_id_by_doc_name(doc_name) for doc_name in req.get("doc_names", [])]
+        doc_ids = DocumentService.get_doc_ids_by_doc_names(req.get("doc_names", []))
         for doc_id in req.get("doc_ids", []):
             if doc_id not in doc_ids:
                 doc_ids.append(doc_id)
@@ -630,11 +671,16 @@ def document_rm():
     FileService.init_knowledgebase_docs(pf_id, tenant_id)
 
     errors = ""
+    docs = DocumentService.get_by_ids(doc_ids)
+    doc_dic = {}
+    for doc in docs:
+        doc_dic[doc.id] = doc
+
     for doc_id in doc_ids:
         try:
-            e, doc = DocumentService.get_by_id(doc_id)
-            if not e:
+            if doc_id not in doc_dic:
                 return get_data_error_result(message="Document not found!")
+            doc = doc_dic[doc_id]
             tenant_id = DocumentService.get_tenant_id(doc_id)
             if not tenant_id:
                 return get_data_error_result(message="Tenant not found!")
@@ -659,7 +705,7 @@ def document_rm():
     return get_json_result(data=True)
 
 
-@manager.route('/completion_aibotk', methods=['POST'])
+@manager.route('/completion_aibotk', methods=['POST'])  # noqa: F821
 @validate_request("Authorization", "conversation_id", "word")
 def completion_faq():
     import base64
@@ -669,16 +715,18 @@ def completion_faq():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
 
     e, conv = API4ConversationService.get_by_id(req["conversation_id"])
     if not e:
         return get_data_error_result(message="Conversation not found!")
-    if "quote" not in req: req["quote"] = True
+    if "quote" not in req:
+        req["quote"] = True
 
     msg = []
     msg.append({"role": "user", "content": req["word"]})
-    if not msg[-1].get("id"): msg[-1]["id"] = get_uuid()
+    if not msg[-1].get("id"):
+        msg[-1]["id"] = get_uuid()
     message_id = msg[-1]["id"]
 
     def fillin_conv(ans):
@@ -799,24 +847,25 @@ def completion_faq():
         return server_error_response(e)
 
 
-@manager.route('/retrieval', methods=['POST'])
+@manager.route('/retrieval', methods=['POST'])  # noqa: F821
 @validate_request("kb_id", "question")
 def retrieval():
     token = request.headers.get('Authorization').split()[1]
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Token is not valid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
     kb_ids = req.get("kb_id", [])
     doc_ids = req.get("doc_ids", [])
     question = req.get("question")
     page = int(req.get("page", 1))
-    size = int(req.get("size", 30))
+    size = int(req.get("page_size", 30))
     similarity_threshold = float(req.get("similarity_threshold", 0.2))
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
     top = int(req.get("top_k", 1024))
+    highlight = bool(req.get("highlight", False)) 
 
     try:
         kbs = KnowledgebaseService.get_by_ids(kb_ids)
@@ -837,7 +886,8 @@ def retrieval():
             question += keyword_extraction(chat_mdl, question)
         ranks = settings.retrievaler.retrieval(question, embd_mdl, kbs[0].tenant_id, kb_ids, page, size,
                                                similarity_threshold, vector_similarity_weight, top,
-                                               doc_ids, rerank_mdl=rerank_mdl)
+                                               doc_ids, rerank_mdl=rerank_mdl, highlight= highlight,
+                                               rank_feature=label_question(question, kbs))
         for c in ranks["chunks"]:
             c.pop("vector", None)
         return get_json_result(data=ranks)

api/apps/auth/README.md

@@ -0,0 +1,76 @@
+# Auth
+
+The Auth module provides implementations of OAuth2 and OpenID Connect (OIDC) authentication for integration with third-party identity providers. 
+
+**Features**
+
+- Supports both OAuth2 and OIDC authentication protocols
+- Automatic OIDC configuration discovery (via `/.well-known/openid-configuration`)
+- JWT token validation
+- Unified user information handling
+
+## Usage
+
+```python
+# OAuth2 configuration
+oauth_config = {
+    "type": "oauth2",
+    "client_id": "your_client_id",
+    "client_secret": "your_client_secret",
+    "authorization_url": "https://your-oauth-provider.com/oauth/authorize",
+    "token_url": "https://your-oauth-provider.com/oauth/token",
+    "userinfo_url": "https://your-oauth-provider.com/oauth/userinfo",
+    "redirect_uri": "https://your-app.com/v1/user/oauth/callback/<channel>"
+}
+
+# OIDC configuration
+oidc_config = {
+    "type": "oidc",
+    "issuer": "https://your-oauth-provider.com/oidc",
+    "client_id": "your_client_id",
+    "client_secret": "your_client_secret",
+    "redirect_uri": "https://your-app.com/v1/user/oauth/callback/<channel>"
+}
+
+# Github OAuth configuration
+github_config = {
+    "type": "github"
+    "client_id": "your_client_id",
+    "client_secret": "your_client_secret",
+    "redirect_uri": "https://your-app.com/v1/user/oauth/callback/<channel>"
+}
+
+# Get client instance
+client = get_auth_client(oauth_config)
+```
+
+### Authentication Flow
+
+1. Get authorization URL:
+```python
+auth_url = client.get_authorization_url()
+```
+
+2. After user authorization, exchange authorization code for token:
+```python
+token_response = client.exchange_code_for_token(authorization_code)
+access_token = token_response["access_token"]
+```
+
+3. Fetch user information:
+```python
+user_info = client.fetch_user_info(access_token)
+```
+
+## User Information Structure
+
+All authentication methods return user information following this structure:
+
+```python
+{
+    "email": "user@example.com",
+    "username": "username",
+    "nickname": "User Name",
+    "avatar_url": "https://example.com/avatar.jpg"
+}
+```

api/apps/auth/__init__.py

@@ -0,0 +1,40 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from .oauth import OAuthClient
+from .oidc import OIDCClient
+from .github import GithubOAuthClient
+
+
+CLIENT_TYPES = {
+    "oauth2": OAuthClient,
+    "oidc": OIDCClient,
+    "github": GithubOAuthClient
+}
+
+
+def get_auth_client(config)->OAuthClient:
+    channel_type = str(config.get("type", "")).lower()
+    if channel_type == "":
+        if config.get("issuer"):
+            channel_type = "oidc"
+        else:
+            channel_type = "oauth2"
+    client_class = CLIENT_TYPES.get(channel_type)
+    if not client_class:
+        raise ValueError(f"Unsupported type: {channel_type}")
+
+    return client_class(config)

api/apps/auth/github.py

@@ -0,0 +1,63 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import requests
+from .oauth import OAuthClient, UserInfo
+
+
+class GithubOAuthClient(OAuthClient):
+    def __init__(self, config):
+        """
+        Initialize the GithubOAuthClient with the provider's configuration.
+        """
+        config.update({
+            "authorization_url": "https://github.com/login/oauth/authorize",
+            "token_url": "https://github.com/login/oauth/access_token",
+            "userinfo_url": "https://api.github.com/user",
+            "scope": "user:email"
+        })
+        super().__init__(config)
+
+
+    def fetch_user_info(self, access_token, **kwargs):
+        """
+        Fetch github user info.
+        """
+        user_info = {}
+        try:
+            headers = {"Authorization": f"Bearer {access_token}"}
+            # user info
+            response = requests.get(self.userinfo_url, headers=headers, timeout=self.http_request_timeout)
+            response.raise_for_status()
+            user_info.update(response.json())
+            # email info
+            response = requests.get(self.userinfo_url+"/emails", headers=headers, timeout=self.http_request_timeout)
+            response.raise_for_status()
+            email_info = response.json()
+            user_info["email"] = next(
+                (email for email in email_info if email["primary"]), None
+            )["email"]
+            return self.normalize_user_info(user_info)
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to fetch github user info: {e}")
+
+
+    def normalize_user_info(self, user_info):
+        email = user_info.get("email")
+        username = user_info.get("login", str(email).split("@")[0])
+        nickname = user_info.get("name", username)
+        avatar_url = user_info.get("avatar_url", "")
+        return UserInfo(email=email, username=username, nickname=nickname, avatar_url=avatar_url)

api/apps/auth/oauth.py

@@ -0,0 +1,110 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import requests
+import urllib.parse
+
+
+class UserInfo:
+    def __init__(self, email, username, nickname, avatar_url):
+        self.email = email
+        self.username = username
+        self.nickname = nickname
+        self.avatar_url = avatar_url
+    
+    def to_dict(self):
+        return {key: value for key, value in self.__dict__.items()}
+
+
+class OAuthClient:
+    def __init__(self, config):
+        """
+        Initialize the OAuthClient with the provider's configuration.
+        """
+        self.client_id = config["client_id"]
+        self.client_secret = config["client_secret"]
+        self.authorization_url = config["authorization_url"]
+        self.token_url = config["token_url"]
+        self.userinfo_url = config["userinfo_url"]
+        self.redirect_uri = config["redirect_uri"]
+        self.scope = config.get("scope", None)
+
+        self.http_request_timeout = 7
+
+
+    def get_authorization_url(self, state=None):
+        """
+        Generate the authorization URL for user login.
+        """
+        params = {
+            "client_id": self.client_id,
+            "redirect_uri": self.redirect_uri,
+            "response_type": "code",
+        }
+        if self.scope:
+            params["scope"] = self.scope
+        if state:
+            params["state"] = state
+        authorization_url = f"{self.authorization_url}?{urllib.parse.urlencode(params)}"
+        return authorization_url
+
+
+    def exchange_code_for_token(self, code):
+        """
+        Exchange authorization code for access token.
+        """
+        try:
+            payload = {
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "code": code,
+                "redirect_uri": self.redirect_uri,
+                "grant_type": "authorization_code"
+            }
+            response = requests.post(
+                self.token_url,
+                data=payload,
+                headers={"Accept": "application/json"},
+                timeout=self.http_request_timeout
+            )
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to exchange authorization code for token: {e}")
+
+
+    def fetch_user_info(self, access_token, **kwargs):
+        """
+        Fetch user information using access token.
+        """
+        try:
+            headers = {"Authorization": f"Bearer {access_token}"}
+            response = requests.get(self.userinfo_url, headers=headers, timeout=self.http_request_timeout)
+            response.raise_for_status()
+            user_info = response.json()
+            return self.normalize_user_info(user_info)
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to fetch user info: {e}")
+
+
+    def normalize_user_info(self, user_info):
+        email = user_info.get("email")
+        username = user_info.get("username", str(email).split("@")[0])
+        nickname = user_info.get("nickname", username)
+        avatar_url = user_info.get("avatar_url", None)
+        if avatar_url is None:
+            avatar_url = user_info.get("picture", "")
+        return UserInfo(email=email, username=username, nickname=nickname, avatar_url=avatar_url)

api/apps/auth/oidc.py

@@ -0,0 +1,100 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import jwt
+import requests
+from .oauth import OAuthClient
+
+
+class OIDCClient(OAuthClient):
+    def __init__(self, config):
+        """
+        Initialize the OIDCClient with the provider's configuration.
+        Use `issuer` as the single source of truth for configuration discovery.
+        """
+        self.issuer = config.get("issuer")
+        if not self.issuer:
+            raise ValueError("Missing issuer in configuration.")
+
+        oidc_metadata = self._load_oidc_metadata(self.issuer)
+        config.update({
+            'issuer': oidc_metadata['issuer'],
+            'jwks_uri': oidc_metadata['jwks_uri'], 
+            'authorization_url': oidc_metadata['authorization_endpoint'],
+            'token_url': oidc_metadata['token_endpoint'],
+            'userinfo_url': oidc_metadata['userinfo_endpoint']
+        })
+
+        super().__init__(config)
+        self.issuer = config['issuer']
+        self.jwks_uri = config['jwks_uri']
+
+
+    def _load_oidc_metadata(self, issuer):
+        """
+        Load OIDC metadata from `/.well-known/openid-configuration`.
+        """
+        try:
+            metadata_url = f"{issuer}/.well-known/openid-configuration"
+            response = requests.get(metadata_url, timeout=7)
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to fetch OIDC metadata: {e}")
+
+
+    def parse_id_token(self, id_token):
+        """
+        Parse and validate OIDC ID Token (JWT format) with signature verification.
+        """
+        try:
+            # Decode JWT header without verifying signature
+            headers = jwt.get_unverified_header(id_token)
+            
+            # OIDC usually uses `RS256` for signing
+            alg = headers.get("alg", "RS256")
+
+            # Use PyJWT's PyJWKClient to fetch JWKS and find signing key
+            jwks_url = f"{self.issuer}/.well-known/jwks.json"
+            jwks_cli = jwt.PyJWKClient(jwks_url)
+            signing_key = jwks_cli.get_signing_key_from_jwt(id_token).key
+
+            # Decode and verify signature
+            decoded_token = jwt.decode(
+                id_token,
+                key=signing_key,
+                algorithms=[alg],  
+                audience=str(self.client_id),
+                issuer=self.issuer,
+            )
+            return decoded_token
+        except Exception as e:
+            raise ValueError(f"Error parsing ID Token: {e}")
+
+
+    def fetch_user_info(self, access_token, id_token=None, **kwargs):
+        """
+        Fetch user info.
+        """
+        user_info = {}
+        if id_token:
+            user_info = self.parse_id_token(id_token)
+        user_info.update(super().fetch_user_info(access_token).to_dict())
+        return self.normalize_user_info(user_info)
+
+
+    def normalize_user_info(self, user_info):
+        return super().normalize_user_info(user_info)

api/apps/canvas_app.py

@@ -13,27 +13,28 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import logging
 import json
 import traceback
-from functools import partial
 from flask import request, Response
 from flask_login import login_required, current_user
 from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService
+from api.db.services.user_service import TenantService
+from api.db.services.user_canvas_version import UserCanvasVersionService
 from api.settings import RetCode
 from api.utils import get_uuid
 from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result
 from agent.canvas import Canvas
 from peewee import MySQLDatabase, PostgresqlDatabase
+from api.db.db_models import APIToken
+import time
 
-
-@manager.route('/templates', methods=['GET'])
+@manager.route('/templates', methods=['GET'])  # noqa: F821
 @login_required
 def templates():
     return get_json_result(data=[c.to_dict() for c in CanvasTemplateService.get_all()])
 
 
-@manager.route('/list', methods=['GET'])
+@manager.route('/list', methods=['GET'])  # noqa: F821
 @login_required
 def canvas_list():
     return get_json_result(data=sorted([c.to_dict() for c in \
@@ -41,7 +42,7 @@ def canvas_list():
                            )
 
 
-@manager.route('/rm', methods=['POST'])
+@manager.route('/rm', methods=['POST'])  # noqa: F821
 @validate_request("canvas_ids")
 @login_required
 def rm():
@@ -54,18 +55,18 @@ def rm():
     return get_json_result(data=True)
 
 
-@manager.route('/set', methods=['POST'])
+@manager.route('/set', methods=['POST'])  # noqa: F821
 @validate_request("dsl", "title")
 @login_required
 def save():
     req = request.json
     req["user_id"] = current_user.id
-    if not isinstance(req["dsl"], str): req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
-
+    if not isinstance(req["dsl"], str):
+        req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
     req["dsl"] = json.loads(req["dsl"])
     if "id" not in req:
         if UserCanvasService.query(user_id=current_user.id, title=req["title"].strip()):
-            return server_error_response(ValueError("Duplicated title."))
+            return get_data_error_result(message=f"{req['title'].strip()} already exists.")
         req["id"] = get_uuid()
         if not UserCanvasService.save(**req):
             return get_data_error_result(message="Fail to save canvas.")
@@ -75,24 +76,44 @@ def save():
                 data=False, message='Only owner of canvas authorized for this operation.',
                 code=RetCode.OPERATING_ERROR)
         UserCanvasService.update_by_id(req["id"], req)
+    # save version    
+    UserCanvasVersionService.insert( user_canvas_id=req["id"], dsl=req["dsl"], title="{0}_{1}".format(req["title"], time.strftime("%Y_%m_%d_%H_%M_%S")))
+    UserCanvasVersionService.delete_all_versions(req["id"])
     return get_json_result(data=req)
 
+ 
 
-@manager.route('/get/<canvas_id>', methods=['GET'])
+
+@manager.route('/get/<canvas_id>', methods=['GET'])  # noqa: F821
 @login_required
 def get(canvas_id):
+    e, c = UserCanvasService.get_by_tenant_id(canvas_id)
+    if not e:
+        return get_data_error_result(message="canvas not found.")
+    return get_json_result(data=c)
+
+@manager.route('/getsse/<canvas_id>', methods=['GET'])  # type: ignore # noqa: F821
+def getsse(canvas_id):
+    token = request.headers.get('Authorization').split()
+    if len(token) != 2:
+        return get_data_error_result(message='Authorization is not valid!"')
+    token = token[1]
+    objs = APIToken.query(beta=token)
+    if not objs:
+        return get_data_error_result(message='Authentication error: API key is invalid!"')
     e, c = UserCanvasService.get_by_id(canvas_id)
     if not e:
         return get_data_error_result(message="canvas not found.")
     return get_json_result(data=c.to_dict())
 
 
-@manager.route('/completion', methods=['POST'])
+@manager.route('/completion', methods=['POST'])  # noqa: F821
 @validate_request("id")
 @login_required
 def run():
     req = request.json
     stream = req.get("stream", True)
+    running_hint_text = req.get("running_hint_text", "")
     e, cvs = UserCanvasService.get_by_id(req["id"])
     if not e:
         return get_data_error_result(message="canvas not found.")
@@ -118,7 +139,7 @@ def run():
         def sse():
             nonlocal answer, cvs
             try:
-                for ans in canvas.run(stream=True):
+                for ans in canvas.run(running_hint_text = running_hint_text, stream=True):
                     if ans.get("running_status"):
                         yield "data:" + json.dumps({"code": 0, "message": "",
                                                     "data": {"answer": ans["content"],
@@ -132,12 +153,16 @@ def run():
 
                 canvas.messages.append({"role": "assistant", "content": final_ans["content"], "id": message_id})
                 canvas.history.append(("assistant", final_ans["content"]))
+                if not canvas.path[-1]:
+                    canvas.path.pop(-1)
                 if final_ans.get("reference"):
                     canvas.reference.append(final_ans["reference"])
                 cvs.dsl = json.loads(str(canvas))
                 UserCanvasService.update_by_id(req["id"], cvs.to_dict())
             except Exception as e:
                 cvs.dsl = json.loads(str(canvas))
+                if not canvas.path[-1]:
+                    canvas.path.pop(-1)
                 UserCanvasService.update_by_id(req["id"], cvs.to_dict())
                 traceback.print_exc()
                 yield "data:" + json.dumps({"code": 500, "message": str(e),
@@ -152,8 +177,9 @@ def run():
         resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
         return resp
 
-    for answer in canvas.run(stream=False):
-        if answer.get("running_status"): continue
+    for answer in canvas.run(running_hint_text = running_hint_text, stream=False):
+        if answer.get("running_status"):
+            continue
         final_ans["content"] = "\n".join(answer["content"]) if "content" in answer else ""
         canvas.messages.append({"role": "assistant", "content": final_ans["content"], "id": message_id})
         if final_ans.get("reference"):
@@ -163,7 +189,7 @@ def run():
         return get_json_result(data={"answer": final_ans["content"], "reference": final_ans.get("reference", [])})
 
 
-@manager.route('/reset', methods=['POST'])
+@manager.route('/reset', methods=['POST'])  # noqa: F821
 @validate_request("id")
 @login_required
 def reset():
@@ -186,7 +212,51 @@ def reset():
         return server_error_response(e)
 
 
-@manager.route('/test_db_connect', methods=['POST'])
+@manager.route('/input_elements', methods=['GET'])  # noqa: F821
+@login_required
+def input_elements():
+    cvs_id = request.args.get("id")
+    cpn_id = request.args.get("component_id")
+    try:
+        e, user_canvas = UserCanvasService.get_by_id(cvs_id)
+        if not e:
+            return get_data_error_result(message="canvas not found.")
+        if not UserCanvasService.query(user_id=current_user.id, id=cvs_id):
+            return get_json_result(
+                data=False, message='Only owner of canvas authorized for this operation.',
+                code=RetCode.OPERATING_ERROR)
+
+        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id)
+        return get_json_result(data=canvas.get_component_input_elements(cpn_id))
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route('/debug', methods=['POST'])  # noqa: F821
+@validate_request("id", "component_id", "params")
+@login_required
+def debug():
+    req = request.json
+    for p in req["params"]:
+        assert p.get("key")
+    try:
+        e, user_canvas = UserCanvasService.get_by_id(req["id"])
+        if not e:
+            return get_data_error_result(message="canvas not found.")
+        if not UserCanvasService.query(user_id=current_user.id, id=req["id"]):
+            return get_json_result(
+                data=False, message='Only owner of canvas authorized for this operation.',
+                code=RetCode.OPERATING_ERROR)
+
+        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id)
+        canvas.get_component(req["component_id"])["obj"]._param.debug_inputs = req["params"]
+        df = canvas.get_component(req["component_id"])["obj"].debug()
+        return get_json_result(data=df.to_dict(orient="records"))
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route('/test_db_connect', methods=['POST'])  # noqa: F821
 @validate_request("db_type", "database", "username", "host", "port", "password")
 @login_required
 def test_db_connect():
@@ -198,8 +268,84 @@ def test_db_connect():
         elif req["db_type"] == 'postgresql':
             db = PostgresqlDatabase(req["database"], user=req["username"], host=req["host"], port=req["port"],
                                     password=req["password"])
-        db.connect()
+        elif req["db_type"] == 'mssql':
+            import pyodbc
+            connection_string = (
+                f"DRIVER={{ODBC Driver 17 for SQL Server}};"
+                f"SERVER={req['host']},{req['port']};"
+                f"DATABASE={req['database']};"
+                f"UID={req['username']};"
+                f"PWD={req['password']};"
+            )
+            db = pyodbc.connect(connection_string)
+            cursor = db.cursor()
+            cursor.execute("SELECT 1")
+            cursor.close()
+        else:
+            return server_error_response("Unsupported database type.")
+        if req["db_type"] != 'mssql':
+            db.connect()
         db.close()
+        
         return get_json_result(data="Database Connection Successful!")
     except Exception as e:
         return server_error_response(e)
+#api get list version dsl of canvas
+@manager.route('/getlistversion/<canvas_id>', methods=['GET'])  # noqa: F821
+@login_required
+def getlistversion(canvas_id):
+    try:
+        list =sorted([c.to_dict() for c in UserCanvasVersionService.list_by_canvas_id(canvas_id)], key=lambda x: x["update_time"]*-1)
+        return get_json_result(data=list)
+    except Exception as e:
+        return get_data_error_result(message=f"Error getting history files: {e}")
+#api get version dsl of canvas
+@manager.route('/getversion/<version_id>', methods=['GET'])  # noqa: F821
+@login_required
+def getversion( version_id):
+    try:
+      
+        e, version = UserCanvasVersionService.get_by_id(version_id)
+        if version:
+            return get_json_result(data=version.to_dict())
+    except Exception as e:
+        return get_json_result(data=f"Error getting history file: {e}")
+@manager.route('/listteam', methods=['GET'])  # noqa: F821
+@login_required
+def list_kbs():
+    keywords = request.args.get("keywords", "")
+    page_number = int(request.args.get("page", 1))
+    items_per_page = int(request.args.get("page_size", 150))
+    orderby = request.args.get("orderby", "create_time")
+    desc = request.args.get("desc", True)
+    try:
+        tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
+        kbs, total = UserCanvasService.get_by_tenant_ids(
+            [m["tenant_id"] for m in tenants], current_user.id, page_number,
+            items_per_page, orderby, desc, keywords)
+        return get_json_result(data={"kbs": kbs, "total": total})
+    except Exception as e:
+        return server_error_response(e)
+@manager.route('/setting', methods=['POST'])  # noqa: F821
+@validate_request("id", "title", "permission")
+@login_required
+def setting():
+    req = request.json
+    req["user_id"] = current_user.id
+    e,flow = UserCanvasService.get_by_id(req["id"])
+    if not e:
+        return get_data_error_result(message="canvas not found.")
+    flow = flow.to_dict()
+    flow["title"] = req["title"]
+    if req["description"]:
+        flow["description"] = req["description"]
+    if req["permission"]:
+        flow["permission"] = req["permission"]
+    if req["avatar"]:
+        flow["avatar"] = req["avatar"]
+    if not UserCanvasService.query(user_id=current_user.id, id=req["id"]):
+        return get_json_result(
+            data=False, message='Only owner of canvas authorized for this operation.',
+            code=RetCode.OPERATING_ERROR)
+    num= UserCanvasService.update_by_id(req["id"], flow)
+    return get_json_result(data=num)

api/apps/chunk_app.py

@@ -19,9 +19,11 @@ import json
 from flask import request
 from flask_login import login_required, current_user
 
-from api.db.services.dialog_service import keyword_extraction
 from rag.app.qa import rmPrefix, beAdoc
+from rag.app.tag import label_question
 from rag.nlp import search, rag_tokenizer
+from rag.prompts import keyword_extraction, cross_languages
+from rag.settings import PAGERANK_FLD
 from rag.utils import rmSpace
 from api.db import LLMType, ParserType
 from api.db.services.knowledgebase_service import KnowledgebaseService
@@ -31,11 +33,12 @@ from api.utils.api_utils import server_error_response, get_data_error_result, va
 from api.db.services.document_service import DocumentService
 from api import settings
 from api.utils.api_utils import get_json_result
-import hashlib
+import xxhash
 import re
 
 
-@manager.route('/list', methods=['POST'])
+
+@manager.route('/list', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("doc_id")
 def list_chunk():
@@ -68,9 +71,10 @@ def list_chunk():
                 "doc_id": sres.field[id]["doc_id"],
                 "docnm_kwd": sres.field[id]["docnm_kwd"],
                 "important_kwd": sres.field[id].get("important_kwd", []),
+                "question_kwd": sres.field[id].get("question_kwd", []),
                 "image_id": sres.field[id].get("img_id", ""),
-                "available_int": sres.field[id].get("available_int", 1),
-                "positions": json.loads(sres.field[id].get("position_list", "[]")),
+                "available_int": int(sres.field[id].get("available_int", 1)),
+                "positions": sres.field[id].get("position_int", []),
             }
             assert isinstance(d["positions"], list)
             assert len(d["positions"]) == 0 or (isinstance(d["positions"][0], list) and len(d["positions"][0]) == 5)
@@ -83,7 +87,7 @@ def list_chunk():
         return server_error_response(e)
 
 
-@manager.route('/get', methods=['GET'])
+@manager.route('/get', methods=['GET'])  # noqa: F821
 @login_required
 def get():
     chunk_id = request.args["chunk_id"]
@@ -91,12 +95,14 @@ def get():
         tenants = UserTenantService.query(user_id=current_user.id)
         if not tenants:
             return get_data_error_result(message="Tenant not found!")
-        tenant_id = tenants[0].tenant_id
-
-        kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
-        chunk = settings.docStoreConn.get(chunk_id, search.index_name(tenant_id), kb_ids)
+        for tenant in tenants:
+            kb_ids = KnowledgebaseService.get_kb_ids(tenant.tenant_id)
+            chunk = settings.docStoreConn.get(chunk_id, search.index_name(tenant.tenant_id), kb_ids)
+            if chunk:
+                break
         if chunk is None:
-            return server_error_response("Chunk not found")
+            return server_error_response(Exception("Chunk not found"))
+
         k = []
         for n in chunk.keys():
             if re.search(r"(_vec$|_sm_|_tks|_ltks)", n):
@@ -112,10 +118,9 @@ def get():
         return server_error_response(e)
 
 
-@manager.route('/set', methods=['POST'])
+@manager.route('/set', methods=['POST'])  # noqa: F821
 @login_required
-@validate_request("doc_id", "chunk_id", "content_with_weight",
-                  "important_kwd")
+@validate_request("doc_id", "chunk_id", "content_with_weight")
 def set():
     req = request.json
     d = {
@@ -123,8 +128,16 @@ def set():
         "content_with_weight": req["content_with_weight"]}
     d["content_ltks"] = rag_tokenizer.tokenize(req["content_with_weight"])
     d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
-    d["important_kwd"] = req["important_kwd"]
-    d["important_tks"] = rag_tokenizer.tokenize(" ".join(req["important_kwd"]))
+    if "important_kwd" in req:
+        d["important_kwd"] = req["important_kwd"]
+        d["important_tks"] = rag_tokenizer.tokenize(" ".join(req["important_kwd"]))
+    if "question_kwd" in req:
+        d["question_kwd"] = req["question_kwd"]
+        d["question_tks"] = rag_tokenizer.tokenize("\n".join(req["question_kwd"]))
+    if "tag_kwd" in req:
+        d["tag_kwd"] = req["tag_kwd"]
+    if "tag_feas" in req:
+        d["tag_feas"] = req["tag_feas"]
     if "available_int" in req:
         d["available_int"] = req["available_int"]
 
@@ -145,23 +158,20 @@ def set():
                 t for t in re.split(
                     r"[\n\t]",
                     req["content_with_weight"]) if len(t) > 1]
-            if len(arr) != 2:
-                return get_data_error_result(
-                    message="Q&A must be separated by TAB/ENTER key.")
-            q, a = rmPrefix(arr[0]), rmPrefix(arr[1])
-            d = beAdoc(d, arr[0], arr[1], not any(
+            q, a = rmPrefix(arr[0]), rmPrefix("\n".join(arr[1:]))
+            d = beAdoc(d, q, a, not any(
                 [rag_tokenizer.is_chinese(t) for t in q + a]))
 
-        v, c = embd_mdl.encode([doc.name, req["content_with_weight"]])
+        v, c = embd_mdl.encode([doc.name, req["content_with_weight"] if not d.get("question_kwd") else "\n".join(d["question_kwd"])])
         v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
         d["q_%d_vec" % len(v)] = v.tolist()
-        settings.docStoreConn.insert([d], search.index_name(tenant_id), doc.kb_id)
+        settings.docStoreConn.update({"id": req["chunk_id"]}, d, search.index_name(tenant_id), doc.kb_id)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/switch', methods=['POST'])
+@manager.route('/switch', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("chunk_ids", "available_int", "doc_id")
 def switch():
@@ -181,10 +191,11 @@ def switch():
         return server_error_response(e)
 
 
-@manager.route('/rm', methods=['POST'])
+@manager.route('/rm', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("chunk_ids", "doc_id")
 def rm():
+    from rag.utils.storage_factory import STORAGE_IMPL
     req = request.json
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
@@ -195,24 +206,27 @@ def rm():
         deleted_chunk_ids = req["chunk_ids"]
         chunk_number = len(deleted_chunk_ids)
         DocumentService.decrement_chunk_num(doc.id, doc.kb_id, 1, chunk_number, 0)
+        for cid in deleted_chunk_ids:
+            if STORAGE_IMPL.obj_exist(doc.kb_id, cid):
+                STORAGE_IMPL.rm(doc.kb_id, cid)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/create', methods=['POST'])
+@manager.route('/create', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("doc_id", "content_with_weight")
 def create():
     req = request.json
-    md5 = hashlib.md5()
-    md5.update((req["content_with_weight"] + req["doc_id"]).encode("utf-8"))
-    chunck_id = md5.hexdigest()
+    chunck_id = xxhash.xxh64((req["content_with_weight"] + req["doc_id"]).encode("utf-8")).hexdigest()
     d = {"id": chunck_id, "content_ltks": rag_tokenizer.tokenize(req["content_with_weight"]),
          "content_with_weight": req["content_with_weight"]}
     d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
     d["important_kwd"] = req.get("important_kwd", [])
     d["important_tks"] = rag_tokenizer.tokenize(" ".join(req.get("important_kwd", [])))
+    d["question_kwd"] = req.get("question_kwd", [])
+    d["question_tks"] = rag_tokenizer.tokenize("\n".join(req.get("question_kwd", [])))
     d["create_time"] = str(datetime.datetime.now()).replace("T", " ")[:19]
     d["create_timestamp_flt"] = datetime.datetime.now().timestamp()
 
@@ -222,16 +236,23 @@ def create():
             return get_data_error_result(message="Document not found!")
         d["kb_id"] = [doc.kb_id]
         d["docnm_kwd"] = doc.name
+        d["title_tks"] = rag_tokenizer.tokenize(doc.name)
         d["doc_id"] = doc.id
 
         tenant_id = DocumentService.get_tenant_id(req["doc_id"])
         if not tenant_id:
             return get_data_error_result(message="Tenant not found!")
 
+        e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
+        if not e:
+            return get_data_error_result(message="Knowledgebase not found!")
+        if kb.pagerank:
+            d[PAGERANK_FLD] = kb.pagerank
+
         embd_id = DocumentService.get_embd_id(req["doc_id"])
         embd_mdl = LLMBundle(tenant_id, LLMType.EMBEDDING.value, embd_id)
 
-        v, c = embd_mdl.encode([doc.name, req["content_with_weight"]])
+        v, c = embd_mdl.encode([doc.name, req["content_with_weight"] if not d["question_kwd"] else "\n".join(d["question_kwd"])])
         v = 0.1 * v[0] + 0.9 * v[1]
         d["q_%d_vec" % len(v)] = v.tolist()
         settings.docStoreConn.insert([d], search.index_name(tenant_id), doc.kb_id)
@@ -243,7 +264,7 @@ def create():
         return server_error_response(e)
 
 
-@manager.route('/retrieval_test', methods=['POST'])
+@manager.route('/retrieval_test', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("kb_id", "question")
 def retrieval_test():
@@ -257,7 +278,9 @@ def retrieval_test():
     doc_ids = req.get("doc_ids", [])
     similarity_threshold = float(req.get("similarity_threshold", 0.0))
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
+    use_kg = req.get("use_kg", False)
     top = int(req.get("top_k", 1024))
+    langs = req.get("cross_languages", [])
     tenant_ids = []
 
     try:
@@ -277,6 +300,9 @@ def retrieval_test():
         if not e:
             return get_data_error_result(message="Knowledgebase not found!")
 
+        if langs:
+            question = cross_languages(kb.tenant_id, None, question, langs)
+
         embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
 
         rerank_mdl = None
@@ -287,12 +313,24 @@ def retrieval_test():
             chat_mdl = LLMBundle(kb.tenant_id, LLMType.CHAT)
             question += keyword_extraction(chat_mdl, question)
 
-        retr = settings.retrievaler if kb.parser_id != ParserType.KG else settings.kg_retrievaler
-        ranks = retr.retrieval(question, embd_mdl, tenant_ids, kb_ids, page, size,
+        labels = label_question(question, [kb])
+        ranks = settings.retrievaler.retrieval(question, embd_mdl, tenant_ids, kb_ids, page, size,
                                similarity_threshold, vector_similarity_weight, top,
-                               doc_ids, rerank_mdl=rerank_mdl, highlight=req.get("highlight"))
+                               doc_ids, rerank_mdl=rerank_mdl, highlight=req.get("highlight"),
+                               rank_feature=labels
+                               )
+        if use_kg:
+            ck = settings.kg_retrievaler.retrieval(question,
+                                                   tenant_ids,
+                                                   kb_ids,
+                                                   embd_mdl,
+                                                   LLMBundle(kb.tenant_id, LLMType.CHAT))
+            if ck["content_with_weight"]:
+                ranks["chunks"].insert(0, ck)
+
         for c in ranks["chunks"]:
             c.pop("vector", None)
+        ranks["labels"] = labels
 
         return get_json_result(data=ranks)
     except Exception as e:
@@ -302,7 +340,7 @@ def retrieval_test():
         return server_error_response(e)
 
 
-@manager.route('/knowledge_graph', methods=['GET'])
+@manager.route('/knowledge_graph', methods=['GET'])  # noqa: F821
 @login_required
 def knowledge_graph():
     doc_id = request.args["doc_id"]

api/apps/conversation_app.py

@@ -17,26 +17,35 @@ import json
 import re
 import traceback
 from copy import deepcopy
-from api.db.services.user_service import UserTenantService
-from flask import request, Response
-from flask_login import login_required, current_user
 
-from api.db import LLMType
-from api.db.services.dialog_service import DialogService, ConversationService, chat, ask
-from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.llm_service import LLMBundle, TenantService, TenantLLMService
+import trio
+from flask import Response, request
+from flask_login import current_user, login_required
+
 from api import settings
-from api.utils.api_utils import get_json_result
-from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from graphrag.mind_map_extractor import MindMapExtractor
+from api.db import LLMType
+from api.db.db_models import APIToken
+from api.db.services.conversation_service import ConversationService, structure_answer
+from api.db.services.dialog_service import DialogService, ask, chat
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.llm_service import LLMBundle, TenantService
+from api.db.services.user_service import UserTenantService
+from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request
+from graphrag.general.mind_map_extractor import MindMapExtractor
+from rag.app.tag import label_question
 
 
-@manager.route('/set', methods=['POST'])
+@manager.route("/set", methods=["POST"])  # noqa: F821
 @login_required
 def set_conversation():
     req = request.json
     conv_id = req.get("conversation_id")
     is_new = req.get("is_new")
+    name = req.get("name", "New conversation")
+
+    if len(name) > 255:
+        name = name[0:255]
+
     del req["is_new"]
     if not is_new:
         del req["conversation_id"]
@@ -45,8 +54,7 @@ def set_conversation():
                 return get_data_error_result(message="Conversation not found!")
             e, conv = ConversationService.get_by_id(conv_id)
             if not e:
-                return get_data_error_result(
-                    message="Fail to update a conversation!")
+                return get_data_error_result(message="Fail to update a conversation!")
             conv = conv.to_dict()
             return get_json_result(data=conv)
         except Exception as e:
@@ -56,23 +64,14 @@ def set_conversation():
         e, dia = DialogService.get_by_id(req["dialog_id"])
         if not e:
             return get_data_error_result(message="Dialog not found")
-        conv = {
-            "id": conv_id,
-            "dialog_id": req["dialog_id"],
-            "name": req.get("name", "New conversation"),
-            "message": [{"role": "assistant", "content": dia.prompt_config["prologue"]}]
-        }
+        conv = {"id": conv_id, "dialog_id": req["dialog_id"], "name": name, "message": [{"role": "assistant", "content": dia.prompt_config["prologue"]}]}
         ConversationService.save(**conv)
-        e, conv = ConversationService.get_by_id(conv["id"])
-        if not e:
-            return get_data_error_result(message="Fail to new a conversation!")
-        conv = conv.to_dict()
         return get_json_result(data=conv)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/get', methods=['GET'])
+@manager.route("/get", methods=["GET"])  # noqa: F821
 @login_required
 def get():
     conv_id = request.args["conversation_id"]
@@ -81,20 +80,64 @@ def get():
         if not e:
             return get_data_error_result(message="Conversation not found!")
         tenants = UserTenantService.query(user_id=current_user.id)
+        avatar = None
         for tenant in tenants:
-            if DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id):
+            dialog = DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id)
+            if dialog and len(dialog) > 0:
+                avatar = dialog[0].icon
                 break
         else:
-            return get_json_result(
-                data=False, message='Only owner of conversation authorized for this operation.',
-                code=settings.RetCode.OPERATING_ERROR)
+            return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+
+        def get_value(d, k1, k2):
+            return d.get(k1, d.get(k2))
+
+        for ref in conv.reference:
+            if isinstance(ref, list):
+                continue
+            ref["chunks"] = [
+                {
+                    "id": get_value(ck, "chunk_id", "id"),
+                    "content": get_value(ck, "content", "content_with_weight"),
+                    "document_id": get_value(ck, "doc_id", "document_id"),
+                    "document_name": get_value(ck, "docnm_kwd", "document_name"),
+                    "dataset_id": get_value(ck, "kb_id", "dataset_id"),
+                    "image_id": get_value(ck, "image_id", "img_id"),
+                    "positions": get_value(ck, "positions", "position_int"),
+                    "doc_type": get_value(ck, "doc_type", "doc_type_kwd"),
+                }
+                for ck in ref.get("chunks", [])
+            ]
+
         conv = conv.to_dict()
+        conv["avatar"] = avatar
         return get_json_result(data=conv)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/rm', methods=['POST'])
+@manager.route("/getsse/<dialog_id>", methods=["GET"])  # type: ignore # noqa: F821
+def getsse(dialog_id):
+    token = request.headers.get("Authorization").split()
+    if len(token) != 2:
+        return get_data_error_result(message='Authorization is not valid!"')
+    token = token[1]
+    objs = APIToken.query(beta=token)
+    if not objs:
+        return get_data_error_result(message='Authentication error: API key is invalid!"')
+    try:
+        e, conv = DialogService.get_by_id(dialog_id)
+        if not e:
+            return get_data_error_result(message="Dialog not found!")
+        conv = conv.to_dict()
+        conv["avatar"] = conv["icon"]
+        del conv["icon"]
+        return get_json_result(data=conv)
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/rm", methods=["POST"])  # noqa: F821
 @login_required
 def rm():
     conv_ids = request.json["conversation_ids"]
@@ -108,35 +151,29 @@ def rm():
                 if DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id):
                     break
             else:
-                return get_json_result(
-                    data=False, message='Only owner of conversation authorized for this operation.',
-                    code=settings.RetCode.OPERATING_ERROR)
+                return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
             ConversationService.delete_by_id(cid)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/list', methods=['GET'])
+@manager.route("/list", methods=["GET"])  # noqa: F821
 @login_required
 def list_convsersation():
     dialog_id = request.args["dialog_id"]
     try:
         if not DialogService.query(tenant_id=current_user.id, id=dialog_id):
-            return get_json_result(
-                data=False, message='Only owner of dialog authorized for this operation.',
-                code=settings.RetCode.OPERATING_ERROR)
-        convs = ConversationService.query(
-            dialog_id=dialog_id,
-            order_by=ConversationService.model.create_time,
-            reverse=True)
+            return get_json_result(data=False, message="Only owner of dialog authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+        convs = ConversationService.query(dialog_id=dialog_id, order_by=ConversationService.model.create_time, reverse=True)
+
         convs = [d.to_dict() for d in convs]
         return get_json_result(data=convs)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/completion', methods=['POST'])
+@manager.route("/completion", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("conversation_id", "messages")
 def completion():
@@ -162,31 +199,42 @@ def completion():
 
         if not conv.reference:
             conv.reference = []
-        conv.message.append({"role": "assistant", "content": "", "id": message_id})
-        conv.reference.append({"chunks": [], "doc_aggs": []})
+        else:
 
-        def fillin_conv(ans):
-            nonlocal conv, message_id
-            if not conv.reference:
-                conv.reference.append(ans["reference"])
-            else:
-                conv.reference[-1] = ans["reference"]
-            conv.message[-1] = {"role": "assistant", "content": ans["answer"],
-                                "id": message_id, "prompt": ans.get("prompt", "")}
-            ans["id"] = message_id
+            def get_value(d, k1, k2):
+                return d.get(k1, d.get(k2))
+
+            for ref in conv.reference:
+                if isinstance(ref, list):
+                    continue
+                ref["chunks"] = [
+                    {
+                        "id": get_value(ck, "chunk_id", "id"),
+                        "content": get_value(ck, "content", "content_with_weight"),
+                        "document_id": get_value(ck, "doc_id", "document_id"),
+                        "document_name": get_value(ck, "docnm_kwd", "document_name"),
+                        "dataset_id": get_value(ck, "kb_id", "dataset_id"),
+                        "image_id": get_value(ck, "image_id", "img_id"),
+                        "positions": get_value(ck, "positions", "position_int"),
+                        "doc_type": get_value(ck, "doc_type_kwd", "doc_type_kwd"),
+                    }
+                    for ck in ref.get("chunks", [])
+                ]
+
+        if not conv.reference:
+            conv.reference = []
+        conv.reference.append({"chunks": [], "doc_aggs": []})
 
         def stream():
             nonlocal dia, msg, req, conv
             try:
                 for ans in chat(dia, msg, True, **req):
-                    fillin_conv(ans)
+                    ans = structure_answer(conv, ans, message_id, conv.id)
                     yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
                 ConversationService.update_by_id(conv.id, conv.to_dict())
             except Exception as e:
                 traceback.print_exc()
-                yield "data:" + json.dumps({"code": 500, "message": str(e),
-                                            "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
-                                           ensure_ascii=False) + "\n\n"
+                yield "data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}}, ensure_ascii=False) + "\n\n"
             yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
 
         if req.get("stream", True):
@@ -200,8 +248,7 @@ def completion():
         else:
             answer = None
             for ans in chat(dia, msg, **req):
-                answer = ans
-                fillin_conv(ans)
+                answer = structure_answer(conv, ans, message_id, req["conversation_id"])
                 ConversationService.update_by_id(conv.id, conv.to_dict())
                 break
             return get_json_result(data=answer)
@@ -209,7 +256,7 @@ def completion():
         return server_error_response(e)
 
 
-@manager.route('/tts', methods=['POST'])
+@manager.route("/tts", methods=["POST"])  # noqa: F821
 @login_required
 def tts():
     req = request.json
@@ -231,9 +278,7 @@ def tts():
                 for chunk in tts_mdl.tts(txt):
                     yield chunk
         except Exception as e:
-            yield ("data:" + json.dumps({"code": 500, "message": str(e),
-                                         "data": {"answer": "**ERROR**: " + str(e)}},
-                                        ensure_ascii=False)).encode('utf-8')
+            yield ("data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e)}}, ensure_ascii=False)).encode("utf-8")
 
     resp = Response(stream_audio(), mimetype="audio/mpeg")
     resp.headers.add_header("Cache-Control", "no-cache")
@@ -243,7 +288,7 @@ def tts():
     return resp
 
 
-@manager.route('/delete_msg', methods=['POST'])
+@manager.route("/delete_msg", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("conversation_id", "message_id")
 def delete_msg():
@@ -266,7 +311,7 @@ def delete_msg():
     return get_json_result(data=conv)
 
 
-@manager.route('/thumbup', methods=['POST'])
+@manager.route("/thumbup", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("conversation_id", "message_id")
 def thumbup():
@@ -274,24 +319,26 @@ def thumbup():
     e, conv = ConversationService.get_by_id(req["conversation_id"])
     if not e:
         return get_data_error_result(message="Conversation not found!")
-    up_down = req.get("set")
+    up_down = req.get("thumbup")
     feedback = req.get("feedback", "")
     conv = conv.to_dict()
     for i, msg in enumerate(conv["message"]):
         if req["message_id"] == msg.get("id", "") and msg.get("role", "") == "assistant":
             if up_down:
                 msg["thumbup"] = True
-                if "feedback" in msg: del msg["feedback"]
+                if "feedback" in msg:
+                    del msg["feedback"]
             else:
                 msg["thumbup"] = False
-                if feedback: msg["feedback"] = feedback
+                if feedback:
+                    msg["feedback"] = feedback
             break
 
     ConversationService.update_by_id(conv["id"], conv)
     return get_json_result(data=conv)
 
 
-@manager.route('/ask', methods=['POST'])
+@manager.route("/ask", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("question", "kb_ids")
 def ask_about():
@@ -304,9 +351,7 @@ def ask_about():
             for ans in ask(req["question"], req["kb_ids"], uid):
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
         except Exception as e:
-            yield "data:" + json.dumps({"code": 500, "message": str(e),
-                                        "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
-                                       ensure_ascii=False) + "\n\n"
+            yield "data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}}, ensure_ascii=False) + "\n\n"
         yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
 
     resp = Response(stream(), mimetype="text/event-stream")
@@ -317,7 +362,7 @@ def ask_about():
     return resp
 
 
-@manager.route('/mindmap', methods=['POST'])
+@manager.route("/mindmap", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("question", "kb_ids")
 def mindmap():
@@ -327,19 +372,19 @@ def mindmap():
     if not e:
         return get_data_error_result(message="Knowledgebase not found!")
 
-    embd_mdl = TenantLLMService.model_instance(
-        kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
+    embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING, llm_name=kb.embd_id)
     chat_mdl = LLMBundle(current_user.id, LLMType.CHAT)
-    ranks = settings.retrievaler.retrieval(req["question"], embd_mdl, kb.tenant_id, kb_ids, 1, 12,
-                                           0.3, 0.3, aggs=False)
+    question = req["question"]
+    ranks = settings.retrievaler.retrieval(question, embd_mdl, kb.tenant_id, kb_ids, 1, 12, 0.3, 0.3, aggs=False, rank_feature=label_question(question, [kb]))
     mindmap = MindMapExtractor(chat_mdl)
-    mind_map = mindmap([c["content_with_weight"] for c in ranks["chunks"]]).output
+    mind_map = trio.run(mindmap, [c["content_with_weight"] for c in ranks["chunks"]])
+    mind_map = mind_map.output
     if "error" in mind_map:
         return server_error_response(Exception(mind_map["error"]))
     return get_json_result(data=mind_map)
 
 
-@manager.route('/related_questions', methods=['POST'])
+@manager.route("/related_questions", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("question")
 def related_questions():
@@ -347,31 +392,49 @@ def related_questions():
     question = req["question"]
     chat_mdl = LLMBundle(current_user.id, LLMType.CHAT)
     prompt = """
-Objective: To generate search terms related to the user's search keywords, helping users find more valuable information.
-Instructions:
- - Based on the keywords provided by the user, generate 5-10 related search terms.
- - Each search term should be directly or indirectly related to the keyword, guiding the user to find more valuable information.
- - Use common, general terms as much as possible, avoiding obscure words or technical jargon.
- - Keep the term length between 2-4 words, concise and clear.
- - DO NOT translate, use the language of the original keywords.
+Role: You are an AI language model assistant tasked with generating 5-10 related questions based on a user’s original query. These questions should help expand the search query scope and improve search relevance.
 
-### Example:
-Keywords: Chinese football
-Related search terms:
-1. Current status of Chinese football
-2. Reform of Chinese football
-3. Youth training of Chinese football
-4. Chinese football in the Asian Cup
-5. Chinese football in the World Cup
+Instructions:
+	Input: You are provided with a user’s question.
+	Output: Generate 5-10 alternative questions that are related to the original user question. These alternatives should help retrieve a broader range of relevant documents from a vector database.
+	Context: Focus on rephrasing the original question in different ways, making sure the alternative questions are diverse but still connected to the topic of the original query. Do not create overly obscure, irrelevant, or unrelated questions.
+	Fallback: If you cannot generate any relevant alternatives, do not return any questions.
+	Guidance:
+	1. Each alternative should be unique but still relevant to the original query.
+	2. Keep the phrasing clear, concise, and easy to understand.
+	3. Avoid overly technical jargon or specialized terms unless directly relevant.
+	4. Ensure that each question contributes towards improving search results by broadening the search angle, not narrowing it.
+
+Example:
+Original Question: What are the benefits of electric vehicles?
+
+Alternative Questions:
+	1. How do electric vehicles impact the environment?
+	2. What are the advantages of owning an electric car?
+	3. What is the cost-effectiveness of electric vehicles?
+	4. How do electric vehicles compare to traditional cars in terms of fuel efficiency?
+	5. What are the environmental benefits of switching to electric cars?
+	6. How do electric vehicles help reduce carbon emissions?
+	7. Why are electric vehicles becoming more popular?
+	8. What are the long-term savings of using electric vehicles?
+	9. How do electric vehicles contribute to sustainability?
+	10. What are the key benefits of electric vehicles for consumers?
 
 Reason:
- - When searching, users often only use one or two keywords, making it difficult to fully express their information needs.
- - Generating related search terms can help users dig deeper into relevant information and improve search efficiency. 
- - At the same time, related terms can also help search engines better understand user needs and return more accurate search results.
- 
+	Rephrasing the original query into multiple alternative questions helps the user explore different aspects of their search topic, improving the quality of search results.
+	These questions guide the search engine to provide a more comprehensive set of relevant documents.
 """
-    ans = chat_mdl.chat(prompt, [{"role": "user", "content": f"""
+    ans = chat_mdl.chat(
+        prompt,
+        [
+            {
+                "role": "user",
+                "content": f"""
 Keywords: {question}
 Related search terms:
-    """}], {"temperature": 0.9})
+    """,
+            }
+        ],
+        {"temperature": 0.9},
+    )
     return get_json_result(data=[re.sub(r"^[0-9]\. ", "", a) for a in ans.split("\n") if re.match(r"^[0-9]\. ", a)])

api/apps/dialog_app.py

@@ -18,6 +18,7 @@ from flask import request
 from flask_login import login_required, current_user
 from api.db.services.dialog_service import DialogService
 from api.db import StatusEnum
+from api.db.services.llm_service import TenantLLMService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import TenantService, UserTenantService
 from api import settings
@@ -26,23 +27,23 @@ from api.utils import get_uuid
 from api.utils.api_utils import get_json_result
 
 
-@manager.route('/set', methods=['POST'])
+@manager.route('/set', methods=['POST'])  # noqa: F821
 @login_required
 def set_dialog():
     req = request.json
     dialog_id = req.get("dialog_id")
     name = req.get("name", "New Dialog")
-    description = req.get("description", "A helpful Dialog")
+    description = req.get("description", "A helpful dialog")
     icon = req.get("icon", "")
     top_n = req.get("top_n", 6)
     top_k = req.get("top_k", 1024)
     rerank_id = req.get("rerank_id", "")
-    if not rerank_id: req["rerank_id"] = ""
+    if not rerank_id:
+        req["rerank_id"] = ""
     similarity_threshold = req.get("similarity_threshold", 0.1)
     vector_similarity_weight = req.get("vector_similarity_weight", 0.3)
-    if vector_similarity_weight is None: vector_similarity_weight = 0.3
     llm_setting = req.get("llm_setting", {})
-    default_prompt = {
+    default_prompt_with_dataset = {
         "system": """你是一个智能助手，请总结知识库的内容来回答问题，请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时，你的回答必须包括“知识库中未找到您要的答案！”这句话。回答需要考虑聊天历史。
 以下是知识库：
 {knowledge}
@@ -53,15 +54,22 @@ def set_dialog():
         ],
         "empty_response": "Sorry! 知识库中未找到相关内容！"
     }
-    prompt_config = req.get("prompt_config", default_prompt)
+    default_prompt_no_dataset = {
+        "system": """You are a helpful assistant.""",
+        "prologue": "您好，我是您的助手小樱，长得可爱又善良，can I help you?",
+        "parameters": [
+           
+        ],
+        "empty_response": ""
+    }
+    prompt_config = req.get("prompt_config", default_prompt_with_dataset)
 
     if not prompt_config["system"]:
-        prompt_config["system"] = default_prompt["system"]
-    # if len(prompt_config["parameters"]) < 1:
-    #     prompt_config["parameters"] = default_prompt["parameters"]
-    # for p in prompt_config["parameters"]:
-    #     if p["key"] == "knowledge":break
-    # else: prompt_config["parameters"].append(default_prompt["parameters"][0])
+        prompt_config["system"] = default_prompt_with_dataset["system"]
+    
+    if not req.get("kb_ids", []):
+        if prompt_config['system'] == default_prompt_with_dataset['system'] or "{knowledge}" in prompt_config['system']:
+            prompt_config = default_prompt_no_dataset
 
     for p in prompt_config["parameters"]:
         if p["optional"]:
@@ -74,16 +82,19 @@ def set_dialog():
         e, tenant = TenantService.get_by_id(current_user.id)
         if not e:
             return get_data_error_result(message="Tenant not found!")
+        kbs = KnowledgebaseService.get_by_ids(req.get("kb_ids", []))
+        embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]  # remove vendor suffix for comparison
+        embd_count = len(set(embd_ids))
+        if embd_count > 1:
+            return get_data_error_result(message=f'Datasets use different embedding models: {[kb.embd_id for kb in kbs]}"')
+
         llm_id = req.get("llm_id", tenant.llm_id)
         if not dialog_id:
-            if not req.get("kb_ids"):
-                return get_data_error_result(
-                    message="Fail! Please select knowledgebase!")
             dia = {
                 "id": get_uuid(),
                 "tenant_id": current_user.id,
                 "name": name,
-                "kb_ids": req["kb_ids"],
+                "kb_ids": req.get("kb_ids", []),
                 "description": description,
                 "llm_id": llm_id,
                 "llm_setting": llm_setting,
@@ -97,10 +108,7 @@ def set_dialog():
             }
             if not DialogService.save(**dia):
                 return get_data_error_result(message="Fail to new a dialog!")
-            e, dia = DialogService.get_by_id(dia["id"])
-            if not e:
-                return get_data_error_result(message="Fail to new a dialog!")
-            return get_json_result(data=dia.to_json())
+            return get_json_result(data=dia)
         else:
             del req["dialog_id"]
             if "kb_names" in req:
@@ -111,13 +119,14 @@ def set_dialog():
             if not e:
                 return get_data_error_result(message="Fail to update a dialog!")
             dia = dia.to_dict()
+            dia.update(req)
             dia["kb_ids"], dia["kb_names"] = get_kb_names(dia["kb_ids"])
             return get_json_result(data=dia)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/get', methods=['GET'])
+@manager.route('/get', methods=['GET'])  # noqa: F821
 @login_required
 def get():
     dialog_id = request.args["dialog_id"]
@@ -143,7 +152,7 @@ def get_kb_names(kb_ids):
     return ids, nms
 
 
-@manager.route('/list', methods=['GET'])
+@manager.route('/list', methods=['GET'])  # noqa: F821
 @login_required
 def list_dialogs():
     try:
@@ -160,7 +169,7 @@ def list_dialogs():
         return server_error_response(e)
 
 
-@manager.route('/rm', methods=['POST'])
+@manager.route('/rm', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("dialog_ids")
 def rm():

api/apps/document_app.py

@@ -20,77 +20,80 @@ import re
 
 import flask
 from flask import request
-from flask_login import login_required, current_user
+from flask_login import current_user, login_required
 
-from api.db.db_models import Task, File
+from api import settings
+from api.constants import IMG_BASE64_PREFIX
+from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileSource, FileType, ParserType, TaskStatus
+from api.db.db_models import File, Task
+from api.db.services import duplicate_name
+from api.db.services.document_service import DocumentService, doc_upload_and_parse
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
+from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import TaskService, queue_tasks
 from api.db.services.user_service import UserTenantService
+from api.utils import get_uuid
+from api.utils.api_utils import (
+    get_data_error_result,
+    get_json_result,
+    server_error_response,
+    validate_request,
+)
+from api.utils.file_utils import filename_type, get_project_base_directory, thumbnail
+from api.utils.web_utils import html2pdf, is_valid_url
 from deepdoc.parser.html_parser import RAGFlowHtmlParser
 from rag.nlp import search
-from api.db.services import duplicate_name
-from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.utils import get_uuid
-from api.db import FileType, TaskStatus, ParserType, FileSource
-from api.db.services.document_service import DocumentService, doc_upload_and_parse
-from api import settings
-from api.utils.api_utils import get_json_result
 from rag.utils.storage_factory import STORAGE_IMPL
-from api.utils.file_utils import filename_type, thumbnail, get_project_base_directory
-from api.utils.web_utils import html2pdf, is_valid_url
-from api.constants import IMG_BASE64_PREFIX
 
 
-@manager.route('/upload', methods=['POST'])
+@manager.route("/upload", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("kb_id")
 def upload():
     kb_id = request.form.get("kb_id")
     if not kb_id:
-        return get_json_result(
-            data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
-    if 'file' not in request.files:
-        return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+    if "file" not in request.files:
+        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
 
-    file_objs = request.files.getlist('file')
+    file_objs = request.files.getlist("file")
     for file_obj in file_objs:
-        if file_obj.filename == '':
-            return get_json_result(
-                data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
+        if file_obj.filename == "":
+            return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
 
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
+    err, files = FileService.upload_document(kb, file_objs, current_user.id)
+
+    if not files:
+        return get_json_result(data=files, message="There seems to be an issue with your file format. Please verify it is correct and not corrupted.", code=settings.RetCode.DATA_ERROR)
+    files = [f[0] for f in files]  # remove the blob
 
-    err, _ = FileService.upload_document(kb, file_objs, current_user.id)
     if err:
-        return get_json_result(
-            data=False, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
-    return get_json_result(data=True)
+        return get_json_result(data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
+    return get_json_result(data=files)
 
 
-@manager.route('/web_crawl', methods=['POST'])
+@manager.route("/web_crawl", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("kb_id", "name", "url")
 def web_crawl():
     kb_id = request.form.get("kb_id")
     if not kb_id:
-        return get_json_result(
-            data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
     name = request.form.get("name")
     url = request.form.get("url")
     if not is_valid_url(url):
-        return get_json_result(
-            data=False, message='The URL format is invalid', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR)
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
 
     blob = html2pdf(url)
-    if not blob: return server_error_response(ValueError("Download failure."))
+    if not blob:
+        return server_error_response(ValueError("Download failure."))
 
     root_folder = FileService.get_root_folder(current_user.id)
     pf_id = root_folder["id"]
@@ -99,10 +102,7 @@ def web_crawl():
     kb_folder = FileService.new_a_file_from_kb(kb.tenant_id, kb.name, kb_root_folder["id"])
 
     try:
-        filename = duplicate_name(
-            DocumentService.query,
-            name=name + ".pdf",
-            kb_id=kb.id)
+        filename = duplicate_name(DocumentService.query, name=name + ".pdf", kb_id=kb.id)
         filetype = filename_type(filename)
         if filetype == FileType.OTHER.value:
             raise RuntimeError("This type of file has not been supported yet!")
@@ -121,7 +121,7 @@ def web_crawl():
             "name": filename,
             "location": location,
             "size": len(blob),
-            "thumbnail": thumbnail(filename, blob)
+            "thumbnail": thumbnail(filename, blob),
         }
         if doc["type"] == FileType.VISUAL:
             doc["parser_id"] = ParserType.PICTURE.value
@@ -138,129 +138,127 @@ def web_crawl():
     return get_json_result(data=True)
 
 
-@manager.route('/create', methods=['POST'])
+@manager.route("/create", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("name", "kb_id")
 def create():
     req = request.json
     kb_id = req["kb_id"]
     if not kb_id:
-        return get_json_result(
-            data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
 
     try:
         e, kb = KnowledgebaseService.get_by_id(kb_id)
         if not e:
-            return get_data_error_result(
-                message="Can't find this knowledgebase!")
+            return get_data_error_result(message="Can't find this knowledgebase!")
 
         if DocumentService.query(name=req["name"], kb_id=kb_id):
-            return get_data_error_result(
-                message="Duplicated document name in the same knowledgebase.")
+            return get_data_error_result(message="Duplicated document name in the same knowledgebase.")
 
-        doc = DocumentService.insert({
-            "id": get_uuid(),
-            "kb_id": kb.id,
-            "parser_id": kb.parser_id,
-            "parser_config": kb.parser_config,
-            "created_by": current_user.id,
-            "type": FileType.VIRTUAL,
-            "name": req["name"],
-            "location": "",
-            "size": 0
-        })
+        doc = DocumentService.insert(
+            {
+                "id": get_uuid(),
+                "kb_id": kb.id,
+                "parser_id": kb.parser_id,
+                "parser_config": kb.parser_config,
+                "created_by": current_user.id,
+                "type": FileType.VIRTUAL,
+                "name": req["name"],
+                "location": "",
+                "size": 0,
+            }
+        )
         return get_json_result(data=doc.to_json())
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/list', methods=['GET'])
+@manager.route("/list", methods=["POST"])  # noqa: F821
 @login_required
 def list_docs():
     kb_id = request.args.get("kb_id")
     if not kb_id:
-        return get_json_result(
-            data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
     tenants = UserTenantService.query(user_id=current_user.id)
     for tenant in tenants:
-        if KnowledgebaseService.query(
-                tenant_id=tenant.tenant_id, id=kb_id):
+        if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
             break
     else:
-        return get_json_result(
-            data=False, message='Only owner of knowledgebase authorized for this operation.',
-            code=settings.RetCode.OPERATING_ERROR)
+        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
     keywords = request.args.get("keywords", "")
 
-    page_number = int(request.args.get("page", 1))
-    items_per_page = int(request.args.get("page_size", 15))
+    page_number = int(request.args.get("page", 0))
+    items_per_page = int(request.args.get("page_size", 0))
     orderby = request.args.get("orderby", "create_time")
     desc = request.args.get("desc", True)
+
+    req = request.get_json()
+
+    run_status = req.get("run_status", [])
+    if run_status:
+        invalid_status = {s for s in run_status if s not in VALID_TASK_STATUS}
+        if invalid_status:
+            return get_data_error_result(message=f"Invalid filter run status conditions: {', '.join(invalid_status)}")
+
+    types = req.get("types", [])
+    if types:
+        invalid_types = {t for t in types if t not in VALID_FILE_TYPES}
+        if invalid_types:
+            return get_data_error_result(message=f"Invalid filter conditions: {', '.join(invalid_types)} type{'s' if len(invalid_types) > 1 else ''}")
+
     try:
-        docs, tol = DocumentService.get_by_kb_id(
-            kb_id, page_number, items_per_page, orderby, desc, keywords)
+        docs, tol = DocumentService.get_by_kb_id(kb_id, page_number, items_per_page, orderby, desc, keywords, run_status, types)
 
         for doc_item in docs:
-            if doc_item['thumbnail'] and not doc_item['thumbnail'].startswith(IMG_BASE64_PREFIX):
-                doc_item['thumbnail'] = f"/v1/document/image/{kb_id}-{doc_item['thumbnail']}"
+            if doc_item["thumbnail"] and not doc_item["thumbnail"].startswith(IMG_BASE64_PREFIX):
+                doc_item["thumbnail"] = f"/v1/document/image/{kb_id}-{doc_item['thumbnail']}"
 
         return get_json_result(data={"total": tol, "docs": docs})
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/infos', methods=['POST'])
+@manager.route("/infos", methods=["POST"])  # noqa: F821
 @login_required
 def docinfos():
     req = request.json
     doc_ids = req["doc_ids"]
     for doc_id in doc_ids:
         if not DocumentService.accessible(doc_id, current_user.id):
-            return get_json_result(
-                data=False,
-                message='No authorization.',
-                code=settings.RetCode.AUTHENTICATION_ERROR
-            )
+            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
     docs = DocumentService.get_by_ids(doc_ids)
     return get_json_result(data=list(docs.dicts()))
 
 
-@manager.route('/thumbnails', methods=['GET'])
+@manager.route("/thumbnails", methods=["GET"])  # noqa: F821
 # @login_required
 def thumbnails():
     doc_ids = request.args.get("doc_ids").split(",")
     if not doc_ids:
-        return get_json_result(
-            data=False, message='Lack of "Document ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "Document ID"', code=settings.RetCode.ARGUMENT_ERROR)
 
     try:
         docs = DocumentService.get_thumbnails(doc_ids)
 
         for doc_item in docs:
-            if doc_item['thumbnail'] and not doc_item['thumbnail'].startswith(IMG_BASE64_PREFIX):
-                doc_item['thumbnail'] = f"/v1/document/image/{doc_item['kb_id']}-{doc_item['thumbnail']}"
+            if doc_item["thumbnail"] and not doc_item["thumbnail"].startswith(IMG_BASE64_PREFIX):
+                doc_item["thumbnail"] = f"/v1/document/image/{doc_item['kb_id']}-{doc_item['thumbnail']}"
 
         return get_json_result(data={d["id"]: d["thumbnail"] for d in docs})
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/change_status', methods=['POST'])
+@manager.route("/change_status", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_id", "status")
 def change_status():
     req = request.json
     if str(req["status"]) not in ["0", "1"]:
-        return get_json_result(
-            data=False,
-            message='"Status" must be either 0 or 1!',
-            code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='"Status" must be either 0 or 1!', code=settings.RetCode.ARGUMENT_ERROR)
 
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(
-            data=False,
-            message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
 
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
@@ -268,42 +266,36 @@ def change_status():
             return get_data_error_result(message="Document not found!")
         e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
         if not e:
-            return get_data_error_result(
-                message="Can't find this knowledgebase!")
+            return get_data_error_result(message="Can't find this knowledgebase!")
 
-        if not DocumentService.update_by_id(
-                req["doc_id"], {"status": str(req["status"])}):
-            return get_data_error_result(
-                message="Database error (Document update)!")
+        if not DocumentService.update_by_id(req["doc_id"], {"status": str(req["status"])}):
+            return get_data_error_result(message="Database error (Document update)!")
 
         status = int(req["status"])
-        settings.docStoreConn.update({"doc_id": req["doc_id"]}, {"available_int": status},
-                                     search.index_name(kb.tenant_id), doc.kb_id)
+        settings.docStoreConn.update({"doc_id": req["doc_id"]}, {"available_int": status}, search.index_name(kb.tenant_id), doc.kb_id)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/rm', methods=['POST'])
+@manager.route("/rm", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_id")
 def rm():
     req = request.json
     doc_ids = req["doc_id"]
-    if isinstance(doc_ids, str): doc_ids = [doc_ids]
+    if isinstance(doc_ids, str):
+        doc_ids = [doc_ids]
 
     for doc_id in doc_ids:
         if not DocumentService.accessible4deletion(doc_id, current_user.id):
-            return get_json_result(
-                data=False,
-                message='No authorization.',
-                code=settings.RetCode.AUTHENTICATION_ERROR
-            )
+            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
 
     root_folder = FileService.get_root_folder(current_user.id)
     pf_id = root_folder["id"]
     FileService.init_knowledgebase_docs(pf_id, current_user.id)
     errors = ""
+    kb_table_num_map = {}
     for doc_id in doc_ids:
         try:
             e, doc = DocumentService.get_by_id(doc_id)
@@ -315,15 +307,27 @@ def rm():
 
             b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
 
+            TaskService.filter_delete([Task.doc_id == doc_id])
             if not DocumentService.remove_document(doc, tenant_id):
-                return get_data_error_result(
-                    message="Database error (Document removal)!")
+                return get_data_error_result(message="Database error (Document removal)!")
 
             f2d = File2DocumentService.get_by_document_id(doc_id)
-            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
+            deleted_file_count = 0
+            if f2d:
+                deleted_file_count = FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
             File2DocumentService.delete_by_document_id(doc_id)
+            if deleted_file_count > 0:
+                STORAGE_IMPL.rm(b, n)
 
-            STORAGE_IMPL.rm(b, n)
+            doc_parser = doc.parser_id
+            if doc_parser == ParserType.TABLE:
+                kb_id = doc.kb_id
+                if kb_id not in kb_table_num_map:
+                    counts = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
+                    kb_table_num_map[kb_id] = counts
+                kb_table_num_map[kb_id] -= 1
+                if kb_table_num_map[kb_id] <= 0:
+                    KnowledgebaseService.delete_field_map(kb_id)
         except Exception as e:
             errors += str(e)
 
@@ -333,79 +337,76 @@ def rm():
     return get_json_result(data=True)
 
 
-@manager.route('/run', methods=['POST'])
+@manager.route("/run", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_ids", "run")
 def run():
     req = request.json
     for doc_id in req["doc_ids"]:
         if not DocumentService.accessible(doc_id, current_user.id):
-            return get_json_result(
-                data=False,
-                message='No authorization.',
-                code=settings.RetCode.AUTHENTICATION_ERROR
-            )
+            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
     try:
+        kb_table_num_map = {}
         for id in req["doc_ids"]:
             info = {"run": str(req["run"]), "progress": 0}
-            if str(req["run"]) == TaskStatus.RUNNING.value:
+            if str(req["run"]) == TaskStatus.RUNNING.value and req.get("delete", False):
                 info["progress_msg"] = ""
                 info["chunk_num"] = 0
                 info["token_num"] = 0
             DocumentService.update_by_id(id, info)
-            # if str(req["run"]) == TaskStatus.CANCEL.value:
             tenant_id = DocumentService.get_tenant_id(id)
             if not tenant_id:
                 return get_data_error_result(message="Tenant not found!")
             e, doc = DocumentService.get_by_id(id)
             if not e:
                 return get_data_error_result(message="Document not found!")
-            if settings.docStoreConn.indexExist(search.index_name(tenant_id), doc.kb_id):
-                settings.docStoreConn.delete({"doc_id": id}, search.index_name(tenant_id), doc.kb_id)
+            if req.get("delete", False):
+                TaskService.filter_delete([Task.doc_id == id])
+                if settings.docStoreConn.indexExist(search.index_name(tenant_id), doc.kb_id):
+                    settings.docStoreConn.delete({"doc_id": id}, search.index_name(tenant_id), doc.kb_id)
 
             if str(req["run"]) == TaskStatus.RUNNING.value:
-                TaskService.filter_delete([Task.doc_id == id])
                 e, doc = DocumentService.get_by_id(id)
                 doc = doc.to_dict()
                 doc["tenant_id"] = tenant_id
+
+                doc_parser = doc.get("parser_id", ParserType.NAIVE)
+                if doc_parser == ParserType.TABLE:
+                    kb_id = doc.get("kb_id")
+                    if not kb_id:
+                        continue
+                    if kb_id not in kb_table_num_map:
+                        count = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
+                        kb_table_num_map[kb_id] = count
+                        if kb_table_num_map[kb_id] <= 0:
+                            KnowledgebaseService.delete_field_map(kb_id)
                 bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
-                queue_tasks(doc, bucket, name)
+                queue_tasks(doc, bucket, name, 0)
 
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/rename', methods=['POST'])
+@manager.route("/rename", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_id", "name")
 def rename():
     req = request.json
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(
-            data=False,
-            message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
-        )
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
             return get_data_error_result(message="Document not found!")
-        if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
-                doc.name.lower()).suffix:
-            return get_json_result(
-                data=False,
-                message="The extension of file can't be changed",
-                code=settings.RetCode.ARGUMENT_ERROR)
+        if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
+            return get_json_result(data=False, message="The extension of file can't be changed", code=settings.RetCode.ARGUMENT_ERROR)
         for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
             if d.name == req["name"]:
-                return get_data_error_result(
-                    message="Duplicated document name in the same knowledgebase.")
+                return get_data_error_result(message="Duplicated document name in the same knowledgebase.")
 
-        if not DocumentService.update_by_id(
-                req["doc_id"], {"name": req["name"]}):
-            return get_data_error_result(
-                message="Database error (Document rename)!")
+        if not DocumentService.update_by_id(req["doc_id"], {"name": req["name"]}):
+            return get_data_error_result(message="Database error (Document rename)!")
 
         informs = File2DocumentService.get_by_document_id(req["doc_id"])
         if informs:
@@ -417,7 +418,7 @@ def rename():
         return server_error_response(e)
 
 
-@manager.route('/get/<doc_id>', methods=['GET'])
+@manager.route("/get/<doc_id>", methods=["GET"])  # noqa: F821
 # @login_required
 def get(doc_id):
     try:
@@ -431,29 +432,22 @@ def get(doc_id):
         ext = re.search(r"\.([^.]+)$", doc.name)
         if ext:
             if doc.type == FileType.VISUAL.value:
-                response.headers.set('Content-Type', 'image/%s' % ext.group(1))
+                response.headers.set("Content-Type", "image/%s" % ext.group(1))
             else:
-                response.headers.set(
-                    'Content-Type',
-                    'application/%s' %
-                    ext.group(1))
+                response.headers.set("Content-Type", "application/%s" % ext.group(1))
         return response
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/change_parser', methods=['POST'])
+@manager.route("/change_parser", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("doc_id", "parser_id")
 def change_parser():
     req = request.json
 
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(
-            data=False,
-            message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
-        )
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
@@ -465,21 +459,16 @@ def change_parser():
             else:
                 return get_json_result(data=True)
 
-        if ((doc.type == FileType.VISUAL and req["parser_id"] != "picture")
-                or (re.search(
-                    r"\.(ppt|pptx|pages)$", doc.name) and req["parser_id"] != "presentation")):
+        if (doc.type == FileType.VISUAL and req["parser_id"] != "picture") or (re.search(r"\.(ppt|pptx|pages)$", doc.name) and req["parser_id"] != "presentation"):
             return get_data_error_result(message="Not supported yet!")
 
-        e = DocumentService.update_by_id(doc.id,
-                                         {"parser_id": req["parser_id"], "progress": 0, "progress_msg": "",
-                                          "run": TaskStatus.UNSTART.value})
+        e = DocumentService.update_by_id(doc.id, {"parser_id": req["parser_id"], "progress": 0, "progress_msg": "", "run": TaskStatus.UNSTART.value})
         if not e:
             return get_data_error_result(message="Document not found!")
         if "parser_config" in req:
             DocumentService.update_parser_config(doc.id, req["parser_config"])
         if doc.token_num > 0:
-            e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1,
-                                                    doc.process_duation * -1)
+            e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, doc.process_duation * -1)
             if not e:
                 return get_data_error_result(message="Document not found!")
             tenant_id = DocumentService.get_tenant_id(req["doc_id"])
@@ -493,62 +482,58 @@ def change_parser():
         return server_error_response(e)
 
 
-@manager.route('/image/<image_id>', methods=['GET'])
+@manager.route("/image/<image_id>", methods=["GET"])  # noqa: F821
 # @login_required
 def get_image(image_id):
     try:
+        arr = image_id.split("-")
+        if len(arr) != 2:
+            return get_data_error_result(message="Image not found.")
         bkt, nm = image_id.split("-")
         response = flask.make_response(STORAGE_IMPL.get(bkt, nm))
-        response.headers.set('Content-Type', 'image/JPEG')
+        response.headers.set("Content-Type", "image/JPEG")
         return response
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/upload_and_parse', methods=['POST'])
+@manager.route("/upload_and_parse", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("conversation_id")
 def upload_and_parse():
-    if 'file' not in request.files:
-        return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+    if "file" not in request.files:
+        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
 
-    file_objs = request.files.getlist('file')
+    file_objs = request.files.getlist("file")
     for file_obj in file_objs:
-        if file_obj.filename == '':
-            return get_json_result(
-                data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
+        if file_obj.filename == "":
+            return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
 
     doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, current_user.id)
 
     return get_json_result(data=doc_ids)
 
 
-@manager.route('/parse', methods=['POST'])
+@manager.route("/parse", methods=["POST"])  # noqa: F821
 @login_required
 def parse():
     url = request.json.get("url") if request.json else ""
     if url:
         if not is_valid_url(url):
-            return get_json_result(
-                data=False, message='The URL format is invalid', code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR)
         download_path = os.path.join(get_project_base_directory(), "logs/downloads")
         os.makedirs(download_path, exist_ok=True)
         from seleniumwire.webdriver import Chrome, ChromeOptions
+
         options = ChromeOptions()
-        options.add_argument('--headless')
-        options.add_argument('--disable-gpu')
-        options.add_argument('--no-sandbox')
-        options.add_argument('--disable-dev-shm-usage')
-        options.add_experimental_option('prefs', {
-            'download.default_directory': download_path,
-            'download.prompt_for_download': False,
-            'download.directory_upgrade': True,
-            'safebrowsing.enabled': True
-        })
+        options.add_argument("--headless")
+        options.add_argument("--disable-gpu")
+        options.add_argument("--no-sandbox")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_experimental_option("prefs", {"download.default_directory": download_path, "download.prompt_for_download": False, "download.directory_upgrade": True, "safebrowsing.enabled": True})
         driver = Chrome(options=options)
         driver.get(url)
-        res_headers = [r.response.headers for r in driver.requests]
+        res_headers = [r.response.headers for r in driver.requests if r and r.response]
         if len(res_headers) > 1:
             sections = RAGFlowHtmlParser().parser_txt(driver.page_source)
             driver.quit()
@@ -568,17 +553,42 @@ def parse():
 
         r = re.search(r"filename=\"([^\"]+)\"", str(res_headers))
         if not r or not r.group(1):
-            return get_json_result(
-                data=False, message="Can't not identify downloaded file", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="Can't not identify downloaded file", code=settings.RetCode.ARGUMENT_ERROR)
         f = File(r.group(1), os.path.join(download_path, r.group(1)))
         txt = FileService.parse_docs([f], current_user.id)
         return get_json_result(data=txt)
 
-    if 'file' not in request.files:
-        return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+    if "file" not in request.files:
+        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
 
-    file_objs = request.files.getlist('file')
+    file_objs = request.files.getlist("file")
     txt = FileService.parse_docs(file_objs, current_user.id)
 
     return get_json_result(data=txt)
+
+
+@manager.route("/set_meta", methods=["POST"])  # noqa: F821
+@login_required
+@validate_request("doc_id", "meta")
+def set_meta():
+    req = request.json
+    if not DocumentService.accessible(req["doc_id"], current_user.id):
+        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+    try:
+        meta = json.loads(req["meta"])
+    except Exception as e:
+        return get_json_result(data=False, message=f"Json syntax error: {e}", code=settings.RetCode.ARGUMENT_ERROR)
+    if not isinstance(meta, dict):
+        return get_json_result(data=False, message='Meta data should be in Json map format, like {"key": "value"}', code=settings.RetCode.ARGUMENT_ERROR)
+
+    try:
+        e, doc = DocumentService.get_by_id(req["doc_id"])
+        if not e:
+            return get_data_error_result(message="Document not found!")
+
+        if not DocumentService.update_by_id(req["doc_id"], {"meta_fields": meta}):
+            return get_data_error_result(message="Database error (meta updates)!")
+
+        return get_json_result(data=True)
+    except Exception as e:
+        return server_error_response(e)

api/apps/file2document_app.py

@@ -28,7 +28,7 @@ from api import settings
 from api.utils.api_utils import get_json_result
 
 
-@manager.route('/convert', methods=['POST'])
+@manager.route('/convert', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("file_ids", "kb_ids")
 def convert():
@@ -38,8 +38,12 @@ def convert():
     file2documents = []
 
     try:
+        files = FileService.get_by_ids(file_ids)
+        files_set = dict({file.id: file for file in files})
         for file_id in file_ids:
-            e, file = FileService.get_by_id(file_id)
+            file = files_set[file_id]
+            if not file:
+                return get_data_error_result(message="File not found!")
             file_ids_list = [file_id]
             if file.type == FileType.FOLDER.value:
                 file_ids_list = FileService.get_all_innermost_file_ids(file_id, [])
@@ -86,13 +90,14 @@ def convert():
                         "file_id": id,
                         "document_id": doc.id,
                     })
+
                     file2documents.append(file2document.to_json())
         return get_json_result(data=file2documents)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/rm', methods=['POST'])
+@manager.route('/rm', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("file_ids")
 def rm():

api/apps/file_app.py

@@ -34,7 +34,7 @@ from api.utils.file_utils import filename_type
 from rag.utils.storage_factory import STORAGE_IMPL
 
 
-@manager.route('/upload', methods=['POST'])
+@manager.route('/upload', methods=['POST'])  # noqa: F821
 @login_required
 # @validate_request("parent_id")
 def upload():
@@ -55,20 +55,17 @@ def upload():
                 data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
     file_res = []
     try:
+        e, pf_folder = FileService.get_by_id(pf_id)
+        if not e:
+            return get_data_error_result( message="Can't find this folder!")
         for file_obj in file_objs:
-            e, file = FileService.get_by_id(pf_id)
-            if not e:
-                return get_data_error_result(
-                    message="Can't find this folder!")
             MAX_FILE_NUM_PER_USER = int(os.environ.get('MAX_FILE_NUM_PER_USER', 0))
             if MAX_FILE_NUM_PER_USER > 0 and DocumentService.get_doc_count(current_user.id) >= MAX_FILE_NUM_PER_USER:
-                return get_data_error_result(
-                    message="Exceed the maximum file number of a free user!")
+                return get_data_error_result( message="Exceed the maximum file number of a free user!")
 
             # split file name path
             if not file_obj.filename:
-                e, file = FileService.get_by_id(pf_id)
-                file_obj_names = [file.name, file_obj.filename]
+                file_obj_names = [pf_folder.name, file_obj.filename]
             else:
                 full_path = '/' + file_obj.filename
                 file_obj_names = full_path.split('/')
@@ -120,7 +117,7 @@ def upload():
         return server_error_response(e)
 
 
-@manager.route('/create', methods=['POST'])
+@manager.route('/create', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("name")
 def create():
@@ -160,7 +157,7 @@ def create():
         return server_error_response(e)
 
 
-@manager.route('/list', methods=['GET'])
+@manager.route('/list', methods=['GET'])  # noqa: F821
 @login_required
 def list_files():
     pf_id = request.args.get("parent_id")
@@ -184,7 +181,7 @@ def list_files():
             current_user.id, pf_id, page_number, items_per_page, orderby, desc, keywords)
 
         parent_folder = FileService.get_parent_folder(pf_id)
-        if not FileService.get_parent_folder(pf_id):
+        if not parent_folder:
             return get_json_result(message="File not found!")
 
         return get_json_result(data={"total": total, "files": files, "parent_folder": parent_folder.to_json()})
@@ -192,7 +189,7 @@ def list_files():
         return server_error_response(e)
 
 
-@manager.route('/root_folder', methods=['GET'])
+@manager.route('/root_folder', methods=['GET'])  # noqa: F821
 @login_required
 def get_root_folder():
     try:
@@ -202,7 +199,7 @@ def get_root_folder():
         return server_error_response(e)
 
 
-@manager.route('/parent_folder', methods=['GET'])
+@manager.route('/parent_folder', methods=['GET'])  # noqa: F821
 @login_required
 def get_parent_folder():
     file_id = request.args.get("file_id")
@@ -217,7 +214,7 @@ def get_parent_folder():
         return server_error_response(e)
 
 
-@manager.route('/all_parent_folder', methods=['GET'])
+@manager.route('/all_parent_folder', methods=['GET'])  # noqa: F821
 @login_required
 def get_all_parent_folders():
     file_id = request.args.get("file_id")
@@ -235,7 +232,7 @@ def get_all_parent_folders():
         return server_error_response(e)
 
 
-@manager.route('/rm', methods=['POST'])
+@manager.route('/rm', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("file_ids")
 def rm():
@@ -260,6 +257,7 @@ def rm():
                     STORAGE_IMPL.rm(file.parent_id, file.location)
                 FileService.delete_folder_by_pf_id(current_user.id, file_id)
             else:
+                STORAGE_IMPL.rm(file.parent_id, file.location)
                 if not FileService.delete(file):
                     return get_data_error_result(
                         message="Database error (File removal)!")
@@ -284,7 +282,7 @@ def rm():
         return server_error_response(e)
 
 
-@manager.route('/rename', methods=['POST'])
+@manager.route('/rename', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("file_id", "name")
 def rename():
@@ -322,15 +320,20 @@ def rename():
         return server_error_response(e)
 
 
-@manager.route('/get/<file_id>', methods=['GET'])
-# @login_required
+@manager.route('/get/<file_id>', methods=['GET'])  # noqa: F821
+@login_required
 def get(file_id):
     try:
         e, file = FileService.get_by_id(file_id)
         if not e:
             return get_data_error_result(message="Document not found!")
-        b, n = File2DocumentService.get_storage_address(file_id=file_id)
-        response = flask.make_response(STORAGE_IMPL.get(b, n))
+
+        blob = STORAGE_IMPL.get(file.parent_id, file.location)
+        if not blob:
+            b, n = File2DocumentService.get_storage_address(file_id=file_id)
+            blob = STORAGE_IMPL.get(b, n)
+
+        response = flask.make_response(blob)
         ext = re.search(r"\.([^.]+)$", file.name)
         if ext:
             if file.type == FileType.VISUAL.value:
@@ -345,7 +348,7 @@ def get(file_id):
         return server_error_response(e)
 
 
-@manager.route('/mv', methods=['POST'])
+@manager.route('/mv', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("src_file_ids", "dest_file_id")
 def move():
@@ -353,9 +356,14 @@ def move():
     try:
         file_ids = req["src_file_ids"]
         parent_id = req["dest_file_id"]
+        files = FileService.get_by_ids(file_ids)
+        files_dict = {}
+        for file in files:
+            files_dict[file.id] = file
+
         for file_id in file_ids:
-            e, file = FileService.get_by_id(file_id)
-            if not e:
+            file = files_dict[file_id]
+            if not file:
                 return get_data_error_result(message="File or Folder not found!")
             if not file.tenant_id:
                 return get_data_error_result(message="Tenant not found!")

api/apps/kb_app.py

@@ -13,6 +13,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import json
+import os
+
 from flask import request
 from flask_login import login_required, current_user
 
@@ -21,7 +24,7 @@ from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.user_service import TenantService, UserTenantService
-from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
+from api.utils.api_utils import server_error_response, get_data_error_result, validate_request, not_allowed_parameters
 from api.utils import get_uuid
 from api.db import StatusEnum, FileSource
 from api.db.services.knowledgebase_service import KnowledgebaseService
@@ -29,21 +32,33 @@ from api.db.db_models import File
 from api.utils.api_utils import get_json_result
 from api import settings
 from rag.nlp import search
+from api.constants import DATASET_NAME_LIMIT
+from rag.settings import PAGERANK_FLD
 
 
-@manager.route('/create', methods=['post'])
+@manager.route('/create', methods=['post'])  # noqa: F821
 @login_required
 @validate_request("name")
 def create():
     req = request.json
-    req["name"] = req["name"].strip()
-    req["name"] = duplicate_name(
+    dataset_name = req["name"]
+    if not isinstance(dataset_name, str):
+        return get_data_error_result(message="Dataset name must be string.")
+    if dataset_name == "":
+        return get_data_error_result(message="Dataset name can't be empty.")
+    if len(dataset_name) >= DATASET_NAME_LIMIT:
+        return get_data_error_result(
+            message=f"Dataset name length is {len(dataset_name)} which is large than {DATASET_NAME_LIMIT}")
+
+    dataset_name = dataset_name.strip()
+    dataset_name = duplicate_name(
         KnowledgebaseService.query,
-        name=req["name"],
+        name=dataset_name,
         tenant_id=current_user.id,
         status=StatusEnum.VALID.value)
     try:
         req["id"] = get_uuid()
+        req["name"] = dataset_name
         req["tenant_id"] = current_user.id
         req["created_by"] = current_user.id
         e, t = TenantService.get_by_id(current_user.id)
@@ -57,9 +72,10 @@ def create():
         return server_error_response(e)
 
 
-@manager.route('/update', methods=['post'])
+@manager.route('/update', methods=['post'])  # noqa: F821
 @login_required
-@validate_request("kb_id", "name", "description", "permission", "parser_id")
+@validate_request("kb_id", "name", "description", "parser_id")
+@not_allowed_parameters("id", "tenant_id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by")
 def update():
     req = request.json
     req["name"] = req["name"].strip()
@@ -73,15 +89,24 @@ def update():
         if not KnowledgebaseService.query(
                 created_by=current_user.id, id=req["kb_id"]):
             return get_json_result(
-                data=False, message='Only owner of knowledgebase authorized for this operation.', code=settings.RetCode.OPERATING_ERROR)
+                data=False, message='Only owner of knowledgebase authorized for this operation.',
+                code=settings.RetCode.OPERATING_ERROR)
 
         e, kb = KnowledgebaseService.get_by_id(req["kb_id"])
         if not e:
             return get_data_error_result(
                 message="Can't find this knowledgebase!")
 
+        if req.get("parser_id", "") == "tag" and os.environ.get('DOC_ENGINE', "elasticsearch") == "infinity":
+            return get_json_result(
+                data=False,
+                message='The chunking method Tag has not been supported by Infinity yet.',
+                code=settings.RetCode.OPERATING_ERROR
+            )
+
         if req["name"].lower() != kb.name.lower() \
-                and len(KnowledgebaseService.query(name=req["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) > 1:
+                and len(
+            KnowledgebaseService.query(name=req["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) > 1:
             return get_data_error_result(
                 message="Duplicated knowledgebase name.")
 
@@ -89,17 +114,28 @@ def update():
         if not KnowledgebaseService.update_by_id(kb.id, req):
             return get_data_error_result()
 
+        if kb.pagerank != req.get("pagerank", 0):
+            if req.get("pagerank", 0) > 0:
+                settings.docStoreConn.update({"kb_id": kb.id}, {PAGERANK_FLD: req["pagerank"]},
+                                         search.index_name(kb.tenant_id), kb.id)
+            else:
+                # Elasticsearch requires PAGERANK_FLD be non-zero!
+                settings.docStoreConn.update({"exists": PAGERANK_FLD}, {"remove": PAGERANK_FLD},
+                                         search.index_name(kb.tenant_id), kb.id)
+
         e, kb = KnowledgebaseService.get_by_id(kb.id)
         if not e:
             return get_data_error_result(
                 message="Database error (Knowledgebase rename)!")
+        kb = kb.to_dict()
+        kb.update(req)
 
-        return get_json_result(data=kb.to_json())
+        return get_json_result(data=kb)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/detail', methods=['GET'])
+@manager.route('/detail', methods=['GET'])  # noqa: F821
 @login_required
 def detail():
     kb_id = request.args["kb_id"]
@@ -117,28 +153,45 @@ def detail():
         if not kb:
             return get_data_error_result(
                 message="Can't find this knowledgebase!")
+        kb["size"] = DocumentService.get_total_size_by_kb_id(kb_id=kb["id"],keywords="", run_status=[], types=[])
         return get_json_result(data=kb)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/list', methods=['GET'])
+@manager.route('/list', methods=['POST'])  # noqa: F821
 @login_required
 def list_kbs():
-    page_number = request.args.get("page", 1)
-    items_per_page = request.args.get("page_size", 150)
+    keywords = request.args.get("keywords", "")
+    page_number = int(request.args.get("page", 0))
+    items_per_page = int(request.args.get("page_size", 0))
+    parser_id = request.args.get("parser_id")
     orderby = request.args.get("orderby", "create_time")
     desc = request.args.get("desc", True)
+
+    req = request.get_json()
+    owner_ids = req.get("owner_ids", [])
     try:
-        tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
-        kbs = KnowledgebaseService.get_by_tenant_ids(
-            [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc)
-        return get_json_result(data=kbs)
+        if not owner_ids:
+            tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
+            tenants = [m["tenant_id"] for m in tenants]
+            kbs, total = KnowledgebaseService.get_by_tenant_ids(
+                tenants, current_user.id, page_number,
+                items_per_page, orderby, desc, keywords, parser_id)
+        else:
+            tenants = owner_ids
+            kbs, total = KnowledgebaseService.get_by_tenant_ids(
+                tenants, current_user.id, 0,
+                0, orderby, desc, keywords, parser_id)
+            kbs = [kb for kb in kbs if kb["tenant_id"] in tenants]
+            if page_number and items_per_page:
+                kbs = kbs[(page_number-1)*items_per_page:page_number*items_per_page]
+            total = len(kbs)
+        return get_json_result(data={"kbs": kbs, "total": total})
     except Exception as e:
         return server_error_response(e)
 
-
-@manager.route('/rm', methods=['post'])
+@manager.route('/rm', methods=['post'])  # noqa: F821
 @login_required
 @validate_request("kb_id")
 def rm():
@@ -151,24 +204,151 @@ def rm():
         )
     try:
         kbs = KnowledgebaseService.query(
-                created_by=current_user.id, id=req["kb_id"])
+            created_by=current_user.id, id=req["kb_id"])
         if not kbs:
             return get_json_result(
-                data=False, message='Only owner of knowledgebase authorized for this operation.', code=settings.RetCode.OPERATING_ERROR)
+                data=False, message='Only owner of knowledgebase authorized for this operation.',
+                code=settings.RetCode.OPERATING_ERROR)
 
         for doc in DocumentService.query(kb_id=req["kb_id"]):
             if not DocumentService.remove_document(doc, kbs[0].tenant_id):
                 return get_data_error_result(
                     message="Database error (Document removal)!")
             f2d = File2DocumentService.get_by_document_id(doc.id)
-            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
-            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
+            if f2d:
+                FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
             File2DocumentService.delete_by_document_id(doc.id)
-
+        FileService.filter_delete(
+            [File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
         if not KnowledgebaseService.delete_by_id(req["kb_id"]):
             return get_data_error_result(
                 message="Database error (Knowledgebase removal)!")
-        settings.docStoreConn.delete({"kb_id": req["kb_id"]}, search.index_name(kbs[0].tenant_id), req["kb_id"])
+        for kb in kbs:
+            settings.docStoreConn.delete({"kb_id": kb.id}, search.index_name(kb.tenant_id), kb.id)
+            settings.docStoreConn.deleteIdx(search.index_name(kb.tenant_id), kb.id)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
+
+
+@manager.route('/<kb_id>/tags', methods=['GET'])  # noqa: F821
+@login_required
+def list_tags(kb_id):
+    if not KnowledgebaseService.accessible(kb_id, current_user.id):
+        return get_json_result(
+            data=False,
+            message='No authorization.',
+            code=settings.RetCode.AUTHENTICATION_ERROR
+        )
+
+    tags = settings.retrievaler.all_tags(current_user.id, [kb_id])
+    return get_json_result(data=tags)
+
+
+@manager.route('/tags', methods=['GET'])  # noqa: F821
+@login_required
+def list_tags_from_kbs():
+    kb_ids = request.args.get("kb_ids", "").split(",")
+    for kb_id in kb_ids:
+        if not KnowledgebaseService.accessible(kb_id, current_user.id):
+            return get_json_result(
+                data=False,
+                message='No authorization.',
+                code=settings.RetCode.AUTHENTICATION_ERROR
+            )
+
+    tags = settings.retrievaler.all_tags(current_user.id, kb_ids)
+    return get_json_result(data=tags)
+
+
+@manager.route('/<kb_id>/rm_tags', methods=['POST'])  # noqa: F821
+@login_required
+def rm_tags(kb_id):
+    req = request.json
+    if not KnowledgebaseService.accessible(kb_id, current_user.id):
+        return get_json_result(
+            data=False,
+            message='No authorization.',
+            code=settings.RetCode.AUTHENTICATION_ERROR
+        )
+    e, kb = KnowledgebaseService.get_by_id(kb_id)
+
+    for t in req["tags"]:
+        settings.docStoreConn.update({"tag_kwd": t, "kb_id": [kb_id]},
+                                     {"remove": {"tag_kwd": t}},
+                                     search.index_name(kb.tenant_id),
+                                     kb_id)
+    return get_json_result(data=True)
+
+
+@manager.route('/<kb_id>/rename_tag', methods=['POST'])  # noqa: F821
+@login_required
+def rename_tags(kb_id):
+    req = request.json
+    if not KnowledgebaseService.accessible(kb_id, current_user.id):
+        return get_json_result(
+            data=False,
+            message='No authorization.',
+            code=settings.RetCode.AUTHENTICATION_ERROR
+        )
+    e, kb = KnowledgebaseService.get_by_id(kb_id)
+
+    settings.docStoreConn.update({"tag_kwd": req["from_tag"], "kb_id": [kb_id]},
+                                     {"remove": {"tag_kwd": req["from_tag"].strip()}, "add": {"tag_kwd": req["to_tag"]}},
+                                     search.index_name(kb.tenant_id),
+                                     kb_id)
+    return get_json_result(data=True)
+
+
+@manager.route('/<kb_id>/knowledge_graph', methods=['GET'])  # noqa: F821
+@login_required
+def knowledge_graph(kb_id):
+    if not KnowledgebaseService.accessible(kb_id, current_user.id):
+        return get_json_result(
+            data=False,
+            message='No authorization.',
+            code=settings.RetCode.AUTHENTICATION_ERROR
+        )
+    _, kb = KnowledgebaseService.get_by_id(kb_id)
+    req = {
+        "kb_id": [kb_id],
+        "knowledge_graph_kwd": ["graph"]
+    }
+
+    obj = {"graph": {}, "mind_map": {}}
+    if not settings.docStoreConn.indexExist(search.index_name(kb.tenant_id), kb_id):
+        return get_json_result(data=obj)
+    sres = settings.retrievaler.search(req, search.index_name(kb.tenant_id), [kb_id])
+    if not len(sres.ids):
+        return get_json_result(data=obj)
+
+    for id in sres.ids[:1]:
+        ty = sres.field[id]["knowledge_graph_kwd"]
+        try:
+            content_json = json.loads(sres.field[id]["content_with_weight"])
+        except Exception:
+            continue
+
+        obj[ty] = content_json
+
+    if "nodes" in obj["graph"]:
+        obj["graph"]["nodes"] = sorted(obj["graph"]["nodes"], key=lambda x: x.get("pagerank", 0), reverse=True)[:256]
+        if "edges" in obj["graph"]:
+            node_id_set = { o["id"] for o in obj["graph"]["nodes"] }
+            filtered_edges = [o for o in obj["graph"]["edges"] if o["source"] != o["target"] and o["source"] in node_id_set and o["target"] in node_id_set]
+            obj["graph"]["edges"] = sorted(filtered_edges, key=lambda x: x.get("weight", 0), reverse=True)[:128]
+    return get_json_result(data=obj)
+
+@manager.route('/<kb_id>/knowledge_graph', methods=['DELETE'])  # noqa: F821
+@login_required
+def delete_knowledge_graph(kb_id):
+    if not KnowledgebaseService.accessible(kb_id, current_user.id):
+        return get_json_result(
+            data=False,
+            message='No authorization.',
+            code=settings.RetCode.AUTHENTICATION_ERROR
+        )
+    _, kb = KnowledgebaseService.get_by_id(kb_id)
+    settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), kb_id)
+
+    return get_json_result(data=True)

api/apps/langfuse_app.py

@@ -0,0 +1,97 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+from flask import request
+from flask_login import current_user, login_required
+from langfuse import Langfuse
+
+from api.db.db_models import DB
+from api.db.services.langfuse_service import TenantLangfuseService
+from api.utils.api_utils import get_error_data_result, get_json_result, server_error_response, validate_request
+
+
+@manager.route("/api_key", methods=["POST", "PUT"])  # noqa: F821
+@login_required
+@validate_request("secret_key", "public_key", "host")
+def set_api_key():
+    req = request.get_json()
+    secret_key = req.get("secret_key", "")
+    public_key = req.get("public_key", "")
+    host = req.get("host", "")
+    if not all([secret_key, public_key, host]):
+        return get_error_data_result(message="Missing required fields")
+
+    langfuse_keys = dict(
+        tenant_id=current_user.id,
+        secret_key=secret_key,
+        public_key=public_key,
+        host=host,
+    )
+
+    langfuse = Langfuse(public_key=langfuse_keys["public_key"], secret_key=langfuse_keys["secret_key"], host=langfuse_keys["host"])
+    if not langfuse.auth_check():
+        return get_error_data_result(message="Invalid Langfuse keys")
+
+    langfuse_entry = TenantLangfuseService.filter_by_tenant(tenant_id=current_user.id)
+    with DB.atomic():
+        try:
+            if not langfuse_entry:
+                TenantLangfuseService.save(**langfuse_keys)
+            else:
+                TenantLangfuseService.update_by_tenant(tenant_id=current_user.id, langfuse_keys=langfuse_keys)
+            return get_json_result(data=langfuse_keys)
+        except Exception as e:
+            server_error_response(e)
+
+
+@manager.route("/api_key", methods=["GET"])  # noqa: F821
+@login_required
+@validate_request()
+def get_api_key():
+    langfuse_entry = TenantLangfuseService.filter_by_tenant_with_info(tenant_id=current_user.id)
+    if not langfuse_entry:
+        return get_json_result(message="Have not record any Langfuse keys.")
+
+    langfuse = Langfuse(public_key=langfuse_entry["public_key"], secret_key=langfuse_entry["secret_key"], host=langfuse_entry["host"])
+    try:
+        if not langfuse.auth_check():
+            return get_error_data_result(message="Invalid Langfuse keys loaded")
+    except langfuse.api.core.api_error.ApiError as api_err:
+        return get_json_result(message=f"Error from Langfuse: {api_err}")
+    except Exception as e:
+        server_error_response(e)
+
+    langfuse_entry["project_id"] = langfuse.api.projects.get().dict()["data"][0]["id"]
+    langfuse_entry["project_name"] = langfuse.api.projects.get().dict()["data"][0]["name"]
+
+    return get_json_result(data=langfuse_entry)
+
+
+@manager.route("/api_key", methods=["DELETE"])  # noqa: F821
+@login_required
+@validate_request()
+def delete_api_key():
+    langfuse_entry = TenantLangfuseService.filter_by_tenant(tenant_id=current_user.id)
+    if not langfuse_entry:
+        return get_json_result(message="Have not record any Langfuse keys.")
+
+    with DB.atomic():
+        try:
+            TenantLangfuseService.delete_model(langfuse_entry)
+            return get_json_result(data=True)
+        except Exception as e:
+            server_error_response(e)

api/apps/llm_app.py

@@ -15,7 +15,7 @@
 #
 import logging
 import json
-
+import os
 from flask import request
 from flask_login import login_required, current_user
 from api.db.services.llm_service import LLMFactoriesService, TenantLLMService, LLMService
@@ -24,11 +24,11 @@ from api.utils.api_utils import server_error_response, get_data_error_result, va
 from api.db import StatusEnum, LLMType
 from api.db.db_models import TenantLLM
 from api.utils.api_utils import get_json_result
+from api.utils.file_utils import get_project_base_directory
 from rag.llm import EmbeddingModel, ChatModel, RerankModel, CvModel, TTSModel
-import requests
 
 
-@manager.route('/factories', methods=['GET'])
+@manager.route('/factories', methods=['GET'])  # noqa: F821
 @login_required
 def factories():
     try:
@@ -50,7 +50,7 @@ def factories():
         return server_error_response(e)
 
 
-@manager.route('/set_api_key', methods=['POST'])
+@manager.route('/set_api_key', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("llm_factory", "api_key")
 def set_api_key():
@@ -61,6 +61,7 @@ def set_api_key():
     msg = ""
     for llm in LLMService.query(fid=factory):
         if not embd_passed and llm.model_type == LLMType.EMBEDDING.value:
+            assert factory in EmbeddingModel, f"Embedding model from {factory} is not supported yet."
             mdl = EmbeddingModel[factory](
                 req["api_key"], llm.llm_name, base_url=req.get("base_url"))
             try:
@@ -71,6 +72,7 @@ def set_api_key():
             except Exception as e:
                 msg += f"\nFail to access embedding model({llm.llm_name}) using this api key." + str(e)
         elif not chat_passed and llm.model_type == LLMType.CHAT.value:
+            assert factory in ChatModel, f"Chat model from {factory} is not supported yet."
             mdl = ChatModel[factory](
                 req["api_key"], llm.llm_name, base_url=req.get("base_url"))
             try:
@@ -83,6 +85,7 @@ def set_api_key():
                 msg += f"\nFail to access model({llm.llm_name}) using this api key." + str(
                     e)
         elif not rerank_passed and llm.model_type == LLMType.RERANK:
+            assert factory in RerankModel, f"Re-rank model from {factory} is not supported yet."
             mdl = RerankModel[factory](
                 req["api_key"], llm.llm_name, base_url=req.get("base_url"))
             try:
@@ -129,12 +132,14 @@ def set_api_key():
     return get_json_result(data=True)
 
 
-@manager.route('/add_llm', methods=['POST'])
+@manager.route('/add_llm', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("llm_factory")
 def add_llm():
     req = request.json
     factory = req["llm_factory"]
+    api_key = req.get("api_key", "x")
+    llm_name = req.get("llm_name")
 
     def apikey_json(keys):
         nonlocal req
@@ -143,7 +148,6 @@ def add_llm():
     if factory == "VolcEngine":
         # For VolcEngine, due to its special authentication method
         # Assemble ark_api_key endpoint_id into api_key
-        llm_name = req["llm_name"]
         api_key = apikey_json(["ark_api_key", "endpoint_id"])
 
     elif factory == "Tencent Hunyuan":
@@ -152,52 +156,43 @@ def add_llm():
 
     elif factory == "Tencent Cloud":
         req["api_key"] = apikey_json(["tencent_cloud_sid", "tencent_cloud_sk"])
+        return set_api_key()
 
     elif factory == "Bedrock":
         # For Bedrock, due to its special authentication method
         # Assemble bedrock_ak, bedrock_sk, bedrock_region
-        llm_name = req["llm_name"]
         api_key = apikey_json(["bedrock_ak", "bedrock_sk", "bedrock_region"])
 
     elif factory == "LocalAI":
-        llm_name = req["llm_name"] + "___LocalAI"
-        api_key = "xxxxxxxxxxxxxxx"
+        llm_name += "___LocalAI"
 
     elif factory == "HuggingFace":
-        llm_name = req["llm_name"] + "___HuggingFace"
-        api_key = "xxxxxxxxxxxxxxx"
+        llm_name += "___HuggingFace"
 
     elif factory == "OpenAI-API-Compatible":
-        llm_name = req["llm_name"] + "___OpenAI-API"
-        api_key = req.get("api_key", "xxxxxxxxxxxxxxx")
+        llm_name += "___OpenAI-API"
+
+    elif factory == "VLLM":
+        llm_name += "___VLLM"
 
     elif factory == "XunFei Spark":
-        llm_name = req["llm_name"]
         if req["model_type"] == "chat":
-            api_key = req.get("spark_api_password", "xxxxxxxxxxxxxxx")
+            api_key = req.get("spark_api_password", "")
         elif req["model_type"] == "tts":
             api_key = apikey_json(["spark_app_id", "spark_api_secret", "spark_api_key"])
 
     elif factory == "BaiduYiyan":
-        llm_name = req["llm_name"]
         api_key = apikey_json(["yiyan_ak", "yiyan_sk"])
 
     elif factory == "Fish Audio":
-        llm_name = req["llm_name"]
         api_key = apikey_json(["fish_audio_ak", "fish_audio_refid"])
 
     elif factory == "Google Cloud":
-        llm_name = req["llm_name"]
         api_key = apikey_json(["google_project_id", "google_region", "google_service_account_key"])
 
     elif factory == "Azure-OpenAI":
-        llm_name = req["llm_name"]
         api_key = apikey_json(["api_key", "api_version"])
 
-    else:
-        llm_name = req["llm_name"]
-        api_key = req.get("api_key", "xxxxxxxxxxxxxxx")
-
     llm = {
         "tenant_id": current_user.id,
         "llm_factory": factory,
@@ -209,74 +204,74 @@ def add_llm():
     }
 
     msg = ""
+    mdl_nm = llm["llm_name"].split("___")[0]
     if llm["model_type"] == LLMType.EMBEDDING.value:
+        assert factory in EmbeddingModel, f"Embedding model from {factory} is not supported yet."
         mdl = EmbeddingModel[factory](
             key=llm['api_key'],
-            model_name=llm["llm_name"],
+            model_name=mdl_nm,
             base_url=llm["api_base"])
         try:
             arr, tc = mdl.encode(["Test if the api key is available"])
-            if len(arr[0]) == 0 or tc == 0:
+            if len(arr[0]) == 0:
                 raise Exception("Fail")
         except Exception as e:
-            msg += f"\nFail to access embedding model({llm['llm_name']})." + str(e)
+            msg += f"\nFail to access embedding model({mdl_nm})." + str(e)
     elif llm["model_type"] == LLMType.CHAT.value:
+        assert factory in ChatModel, f"Chat model from {factory} is not supported yet."
         mdl = ChatModel[factory](
             key=llm['api_key'],
-            model_name=llm["llm_name"],
+            model_name=mdl_nm,
             base_url=llm["api_base"]
         )
         try:
             m, tc = mdl.chat(None, [{"role": "user", "content": "Hello! How are you doing!"}], {
                 "temperature": 0.9})
-            if not tc:
+            if not tc and m.find("**ERROR**:") >= 0:
                 raise Exception(m)
         except Exception as e:
-            msg += f"\nFail to access model({llm['llm_name']})." + str(
+            msg += f"\nFail to access model({mdl_nm})." + str(
                 e)
     elif llm["model_type"] == LLMType.RERANK:
-        mdl = RerankModel[factory](
-            key=llm["api_key"],
-            model_name=llm["llm_name"],
-            base_url=llm["api_base"]
-        )
+        assert factory in RerankModel, f"RE-rank model from {factory} is not supported yet."
         try:
+            mdl = RerankModel[factory](
+                key=llm["api_key"],
+                model_name=mdl_nm,
+                base_url=llm["api_base"]
+            )
             arr, tc = mdl.similarity("Hello~ Ragflower!", ["Hi, there!", "Ohh, my friend!"])
-            if len(arr) == 0 or tc == 0:
+            if len(arr) == 0:
                 raise Exception("Not known.")
+        except KeyError:
+            msg += f"{factory} dose not support this model({mdl_nm})"
         except Exception as e:
-            msg += f"\nFail to access model({llm['llm_name']})." + str(
+            msg += f"\nFail to access model({mdl_nm})." + str(
                 e)
     elif llm["model_type"] == LLMType.IMAGE2TEXT.value:
+        assert factory in CvModel, f"Image to text model from {factory} is not supported yet."
         mdl = CvModel[factory](
             key=llm["api_key"],
-            model_name=llm["llm_name"],
+            model_name=mdl_nm,
             base_url=llm["api_base"]
         )
         try:
-            img_url = (
-                "https://upload.wikimedia.org/wikipedia/comm"
-                "ons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/256"
-                "0px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
-            )
-            res = requests.get(img_url)
-            if res.status_code == 200:
-                m, tc = mdl.describe(res.content)
-                if not tc:
+            with open(os.path.join(get_project_base_directory(), "web/src/assets/yay.jpg"), "rb") as f:
+                m, tc = mdl.describe(f.read())
+                if not m and not tc:
                     raise Exception(m)
-            else:
-                pass
         except Exception as e:
-            msg += f"\nFail to access model({llm['llm_name']})." + str(e)
+            msg += f"\nFail to access model({mdl_nm})." + str(e)
     elif llm["model_type"] == LLMType.TTS:
+        assert factory in TTSModel, f"TTS model from {factory} is not supported yet."
         mdl = TTSModel[factory](
-            key=llm["api_key"], model_name=llm["llm_name"], base_url=llm["api_base"]
+            key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"]
         )
         try:
             for resp in mdl.tts("Hello~ Ragflower!"):
                 pass
         except RuntimeError as e:
-            msg += f"\nFail to access model({llm['llm_name']})." + str(e)
+            msg += f"\nFail to access model({mdl_nm})." + str(e)
     else:
         # TODO: check other type of models
         pass
@@ -292,7 +287,7 @@ def add_llm():
     return get_json_result(data=True)
 
 
-@manager.route('/delete_llm', methods=['POST'])
+@manager.route('/delete_llm', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("llm_factory", "llm_name")
 def delete_llm():
@@ -303,7 +298,7 @@ def delete_llm():
     return get_json_result(data=True)
 
 
-@manager.route('/delete_factory', methods=['POST'])
+@manager.route('/delete_factory', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("llm_factory")
 def delete_factory():
@@ -313,7 +308,7 @@ def delete_factory():
     return get_json_result(data=True)
 
 
-@manager.route('/my_llms', methods=['GET'])
+@manager.route('/my_llms', methods=['GET'])  # noqa: F821
 @login_required
 def my_llms():
     try:
@@ -334,10 +329,10 @@ def my_llms():
         return server_error_response(e)
 
 
-@manager.route('/list', methods=['GET'])
+@manager.route('/list', methods=['GET'])  # noqa: F821
 @login_required
 def list_app():
-    self_deploied = ["Youdao", "FastEmbed", "BAAI", "Ollama", "Xinference", "LocalAI", "LM-Studio"]
+    self_deployed = ["Youdao", "FastEmbed", "BAAI", "Ollama", "Xinference", "LocalAI", "LM-Studio", "GPUStack"]
     weighted = ["Youdao", "FastEmbed", "BAAI"] if settings.LIGHTEN != 0 else []
     model_type = request.args.get("model_type")
     try:
@@ -347,12 +342,12 @@ def list_app():
         llms = [m.to_dict()
                 for m in llms if m.status == StatusEnum.VALID.value and m.fid not in weighted]
         for m in llms:
-            m["available"] = m["fid"] in facts or m["llm_name"].lower() == "flag-embedding" or m["fid"] in self_deploied
+            m["available"] = m["fid"] in facts or m["llm_name"].lower() == "flag-embedding" or m["fid"] in self_deployed
 
         llm_set = set([m["llm_name"] + "@" + m["fid"] for m in llms])
         for o in objs:
-            if not o.api_key: continue
-            if o.llm_name + "@" + o.llm_factory in llm_set: continue
+            if o.llm_name + "@" + o.llm_factory in llm_set:
+                continue
             llms.append({"llm_name": o.llm_name, "model_type": o.model_type, "fid": o.llm_factory, "available": True})
 
         res = {}
@@ -365,4 +360,4 @@ def list_app():
 
         return get_json_result(data=res)
     except Exception as e:
-        return server_error_response(e)
+        return server_error_response(e)

api/apps/plugin_app.py

@@ -0,0 +1,12 @@
+from flask import Response
+from flask_login import login_required
+from api.utils.api_utils import get_json_result
+from plugin import GlobalPluginManager
+
+@manager.route('/llm_tools', methods=['GET'])  # noqa: F821
+@login_required
+def llm_tools() -> Response:
+    tools = GlobalPluginManager.get_llm_tools()
+    tools_metadata = [t.get_metadata() for t in tools]
+
+    return get_json_result(data=tools_metadata)

api/apps/sdk/agent.py

@@ -0,0 +1,128 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import json
+import time
+from typing import Any, cast
+from api.db.services.canvas_service import UserCanvasService
+from api.db.services.user_canvas_version import UserCanvasVersionService
+from api.settings import RetCode
+from api.utils import get_uuid
+from api.utils.api_utils import get_data_error_result, get_error_data_result, get_json_result, token_required
+from api.utils.api_utils import get_result
+from flask import request
+
+@manager.route('/agents', methods=['GET'])  # noqa: F821
+@token_required
+def list_agents(tenant_id):
+    id = request.args.get("id")
+    title = request.args.get("title")
+    if id or title:
+        canvas = UserCanvasService.query(id=id, title=title, user_id=tenant_id)
+        if not canvas:
+            return get_error_data_result("The agent doesn't exist.")
+    page_number = int(request.args.get("page", 1))
+    items_per_page = int(request.args.get("page_size", 30))
+    orderby = request.args.get("orderby", "update_time")
+    if request.args.get("desc") == "False" or request.args.get("desc") == "false":
+        desc = False
+    else:
+        desc = True
+    canvas = UserCanvasService.get_list(tenant_id,page_number,items_per_page,orderby,desc,id,title)
+    return get_result(data=canvas)
+
+
+@manager.route("/agents", methods=["POST"])  # noqa: F821
+@token_required
+def create_agent(tenant_id: str):
+    req: dict[str, Any] = cast(dict[str, Any], request.json)
+    req["user_id"] = tenant_id
+
+    if req.get("dsl") is not None:
+        if not isinstance(req["dsl"], str):
+            req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
+
+        req["dsl"] = json.loads(req["dsl"])
+    else:
+        return get_json_result(data=False, message="No DSL data in request.", code=RetCode.ARGUMENT_ERROR)
+
+    if req.get("title") is not None:
+        req["title"] = req["title"].strip()
+    else:
+        return get_json_result(data=False, message="No title in request.", code=RetCode.ARGUMENT_ERROR)
+
+    if UserCanvasService.query(user_id=tenant_id, title=req["title"]):
+        return get_data_error_result(message=f"Agent with title {req['title']} already exists.")
+
+    agent_id = get_uuid()
+    req["id"] = agent_id
+
+    if not UserCanvasService.save(**req):
+        return get_data_error_result(message="Fail to create agent.")
+
+    UserCanvasVersionService.insert(
+        user_canvas_id=agent_id,
+        title="{0}_{1}".format(req["title"], time.strftime("%Y_%m_%d_%H_%M_%S")),
+        dsl=req["dsl"]
+    )
+
+    return get_json_result(data=True)
+
+
+@manager.route("/agents/<agent_id>", methods=["PUT"])  # noqa: F821
+@token_required
+def update_agent(tenant_id: str, agent_id: str):
+    req: dict[str, Any] = {k: v for k, v in cast(dict[str, Any], request.json).items() if v is not None}
+    req["user_id"] = tenant_id
+
+    if req.get("dsl") is not None:
+        if not isinstance(req["dsl"], str):
+            req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
+
+        req["dsl"] = json.loads(req["dsl"])
+    
+    if req.get("title") is not None:
+        req["title"] = req["title"].strip()
+
+    if not UserCanvasService.query(user_id=tenant_id, id=agent_id):
+        return get_json_result(
+            data=False, message="Only owner of canvas authorized for this operation.",
+            code=RetCode.OPERATING_ERROR)
+
+    UserCanvasService.update_by_id(agent_id, req)
+
+    if req.get("dsl") is not None:
+        UserCanvasVersionService.insert(
+            user_canvas_id=agent_id,
+            title="{0}_{1}".format(req["title"], time.strftime("%Y_%m_%d_%H_%M_%S")),
+            dsl=req["dsl"]
+        )
+
+        UserCanvasVersionService.delete_all_versions(agent_id)
+
+    return get_json_result(data=True)
+
+
+@manager.route("/agents/<agent_id>", methods=["DELETE"])  # noqa: F821
+@token_required
+def delete_agent(tenant_id: str, agent_id: str):
+    if not UserCanvasService.query(user_id=tenant_id, id=agent_id):
+        return get_json_result(
+            data=False, message="Only owner of canvas authorized for this operation.",
+            code=RetCode.OPERATING_ERROR)
+
+    UserCanvasService.delete_by_id(agent_id)
+    return get_json_result(data=True)

api/apps/sdk/chat.py

@@ -13,59 +13,57 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
+
 from flask import request
+
 from api import settings
 from api.db import StatusEnum
 from api.db.services.dialog_service import DialogService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.llm_service import  TenantLLMService
+from api.db.services.llm_service import TenantLLMService
 from api.db.services.user_service import TenantService
 from api.utils import get_uuid
-from api.utils.api_utils import get_error_data_result, token_required
-from api.utils.api_utils import get_result
+from api.utils.api_utils import check_duplicate_ids, get_error_data_result, get_result, token_required
 
 
-
-@manager.route('/chats', methods=['POST'])
+@manager.route("/chats", methods=["POST"])  # noqa: F821
 @token_required
 def create(tenant_id):
-    req=request.json
-    ids= req.get("dataset_ids")
-    if not ids:
-        return get_error_data_result(message="`dataset_ids` is required")
+    req = request.json
+    ids = [i for i in req.get("dataset_ids", []) if i]
     for kb_id in ids:
-        kbs = KnowledgebaseService.accessible(kb_id=kb_id,user_id=tenant_id)
+        kbs = KnowledgebaseService.accessible(kb_id=kb_id, user_id=tenant_id)
         if not kbs:
             return get_error_data_result(f"You don't own the dataset {kb_id}")
         kbs = KnowledgebaseService.query(id=kb_id)
         kb = kbs[0]
         if kb.chunk_num == 0:
             return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
-    kbs = KnowledgebaseService.get_by_ids(ids)
-    embd_count = list(set([kb.embd_id for kb in kbs]))
-    if len(embd_count) != 1:
-        return get_result(message='Datasets use different embedding models."',code=settings.RetCode.AUTHENTICATION_ERROR)
+
+    kbs = KnowledgebaseService.get_by_ids(ids) if ids else []
+    embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]  # remove vendor suffix for comparison
+    embd_count = list(set(embd_ids))
+    if len(embd_count) > 1:
+        return get_result(message='Datasets use different embedding models."', code=settings.RetCode.AUTHENTICATION_ERROR)
     req["kb_ids"] = ids
     # llm
     llm = req.get("llm")
     if llm:
         if "model_name" in llm:
             req["llm_id"] = llm.pop("model_name")
-            if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req["llm_id"],model_type="chat"):
-                return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
+            if req.get("llm_id") is not None:
+                llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(req["llm_id"])
+                if not TenantLLMService.query(tenant_id=tenant_id, llm_name=llm_name, llm_factory=llm_factory, model_type="chat"):
+                    return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
         req["llm_setting"] = req.pop("llm")
     e, tenant = TenantService.get_by_id(tenant_id)
     if not e:
         return get_error_data_result(message="Tenant not found!")
     # prompt
     prompt = req.get("prompt")
-    key_mapping = {"parameters": "variables",
-                   "prologue": "opener",
-                   "quote": "show_quote",
-                   "system": "prompt",
-                   "rerank_id": "rerank_model",
-                   "vector_similarity_weight": "keywords_similarity_weight"}
-    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
+    key_mapping = {"parameters": "variables", "prologue": "opener", "quote": "show_quote", "system": "prompt", "rerank_id": "rerank_model", "vector_similarity_weight": "keywords_similarity_weight"}
+    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id", "top_k"]
     if prompt:
         for new_key, old_key in key_mapping.items():
             if old_key in prompt:
@@ -82,7 +80,8 @@ def create(tenant_id):
     req["top_k"] = req.get("top_k", 1024)
     req["rerank_id"] = req.get("rerank_id", "")
     if req.get("rerank_id"):
-        if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req.get("rerank_id"),model_type="rerank"):
+        value_rerank_model = ["BAAI/bge-reranker-v2-m3", "maidalun1020/bce-reranker-base_v1"]
+        if req["rerank_id"] not in value_rerank_model and not TenantLLMService.query(tenant_id=tenant_id, llm_name=req.get("rerank_id"), model_type="rerank"):
             return get_error_data_result(f"`rerank_model` {req.get('rerank_id')} doesn't exist")
     if not req.get("llm_id"):
         req["llm_id"] = tenant.llm_id
@@ -101,24 +100,24 @@ def create(tenant_id):
       {knowledge}
       The above is the knowledge base.""",
         "prologue": "Hi! I'm your assistant, what can I do for you?",
-        "parameters": [
-            {"key": "knowledge", "optional": False}
-        ],
-        "empty_response": "Sorry! No relevant content was found in the knowledge base!"
+        "parameters": [{"key": "knowledge", "optional": False}],
+        "empty_response": "Sorry! No relevant content was found in the knowledge base!",
+        "quote": True,
+        "tts": False,
+        "refine_multiturn": True,
     }
-    key_list_2 = ["system", "prologue", "parameters", "empty_response"]
+    key_list_2 = ["system", "prologue", "parameters", "empty_response", "quote", "tts", "refine_multiturn"]
     if "prompt_config" not in req:
-        req['prompt_config'] = {}
+        req["prompt_config"] = {}
     for key in key_list_2:
-        temp = req['prompt_config'].get(key)
-        if (not temp and key == 'system') or (key not in req["prompt_config"]):
-            req['prompt_config'][key] = default_prompt[key]
-    for p in req['prompt_config']["parameters"]:
+        temp = req["prompt_config"].get(key)
+        if (not temp and key == "system") or (key not in req["prompt_config"]):
+            req["prompt_config"][key] = default_prompt[key]
+    for p in req["prompt_config"]["parameters"]:
         if p["optional"]:
             continue
-        if req['prompt_config']["system"].find("{%s}" % p["key"]) < 0:
-            return get_error_data_result(
-                message="Parameter '{}' is not used".format(p["key"]))
+        if req["prompt_config"]["system"].find("{%s}" % p["key"]) < 0:
+            return get_error_data_result(message="Parameter '{}' is not used".format(p["key"]))
     # save
     if not DialogService.save(**req):
         return get_error_data_result(message="Fail to new a chat!")
@@ -133,10 +132,7 @@ def create(tenant_id):
         renamed_dict[new_key] = value
     res["prompt"] = renamed_dict
     del res["prompt_config"]
-    new_dict = {"similarity_threshold": res["similarity_threshold"],
-                "keywords_similarity_weight": res["vector_similarity_weight"],
-                "top_n": res["top_n"],
-                "rerank_model": res['rerank_id']}
+    new_dict = {"similarity_threshold": res["similarity_threshold"], "keywords_similarity_weight": 1 - res["vector_similarity_weight"], "top_n": res["top_n"], "rerank_model": res["rerank_id"]}
     res["prompt"].update(new_dict)
     for key in key_list:
         del res[key]
@@ -147,56 +143,46 @@ def create(tenant_id):
     res["avatar"] = res.pop("icon")
     return get_result(data=res)
 
-@manager.route('/chats/<chat_id>', methods=['PUT'])
+
+@manager.route("/chats/<chat_id>", methods=["PUT"])  # noqa: F821
 @token_required
-def update(tenant_id,chat_id):
+def update(tenant_id, chat_id):
     if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
-        return get_error_data_result(message='You do not own the chat')
-    req =request.json
+        return get_error_data_result(message="You do not own the chat")
+    req = request.json
     ids = req.get("dataset_ids")
     if "show_quotation" in req:
-        req["do_refer"]=req.pop("show_quotation")
-    if "dataset_ids" in req:
-        if not ids:
-            return get_error_data_result("`datasets` can't be empty")
-        if ids:
-            for kb_id in ids:
-                kbs = KnowledgebaseService.accessible(kb_id=chat_id, user_id=tenant_id)
-                if not kbs:
-                    return get_error_data_result(f"You don't own the dataset {kb_id}")
-                kbs = KnowledgebaseService.query(id=kb_id)
-                kb = kbs[0]
-                if kb.chunk_num == 0:
-                    return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
-            kbs = KnowledgebaseService.get_by_ids(ids)
-            embd_count=list(set([kb.embd_id for kb in kbs]))
-            if len(embd_count) != 1 :
-                return get_result(
-                    message='Datasets use different embedding models."',
-                    code=settings.RetCode.AUTHENTICATION_ERROR)
-            req["kb_ids"] = ids
+        req["do_refer"] = req.pop("show_quotation")
+    if ids is not None:
+        for kb_id in ids:
+            kbs = KnowledgebaseService.accessible(kb_id=kb_id, user_id=tenant_id)
+            if not kbs:
+                return get_error_data_result(f"You don't own the dataset {kb_id}")
+            kbs = KnowledgebaseService.query(id=kb_id)
+            kb = kbs[0]
+            if kb.chunk_num == 0:
+                return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
+
+        kbs = KnowledgebaseService.get_by_ids(ids)
+        embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]  # remove vendor suffix for comparison
+        embd_count = list(set(embd_ids))
+        if len(embd_count) != 1:
+            return get_result(message='Datasets use different embedding models."', code=settings.RetCode.AUTHENTICATION_ERROR)
+        req["kb_ids"] = ids
     llm = req.get("llm")
     if llm:
         if "model_name" in llm:
             req["llm_id"] = llm.pop("model_name")
-            if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req["llm_id"],model_type="chat"):
+            if not TenantLLMService.query(tenant_id=tenant_id, llm_name=req["llm_id"], model_type="chat"):
                 return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
         req["llm_setting"] = req.pop("llm")
     e, tenant = TenantService.get_by_id(tenant_id)
     if not e:
         return get_error_data_result(message="Tenant not found!")
-    if req.get("rerank_model"):
-        if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req.get("rerank_model"),model_type="rerank"):
-            return get_error_data_result(f"`rerank_model` {req.get('rerank_model')} doesn't exist")
     # prompt
     prompt = req.get("prompt")
-    key_mapping = {"parameters": "variables",
-                   "prologue": "opener",
-                   "quote": "show_quote",
-                   "system": "prompt",
-                   "rerank_id": "rerank_model",
-                   "vector_similarity_weight": "keywords_similarity_weight"}
-    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
+    key_mapping = {"parameters": "variables", "prologue": "opener", "quote": "show_quote", "system": "prompt", "rerank_id": "rerank_model", "vector_similarity_weight": "keywords_similarity_weight"}
+    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id", "top_k"]
     if prompt:
         for new_key, old_key in key_mapping.items():
             if old_key in prompt:
@@ -207,13 +193,15 @@ def update(tenant_id,chat_id):
         req["prompt_config"] = req.pop("prompt")
     e, res = DialogService.get_by_id(chat_id)
     res = res.to_json()
+    if req.get("rerank_id"):
+        value_rerank_model = ["BAAI/bge-reranker-v2-m3", "maidalun1020/bce-reranker-base_v1"]
+        if req["rerank_id"] not in value_rerank_model and not TenantLLMService.query(tenant_id=tenant_id, llm_name=req.get("rerank_id"), model_type="rerank"):
+            return get_error_data_result(f"`rerank_model` {req.get('rerank_id')} doesn't exist")
     if "name" in req:
         if not req.get("name"):
-            return get_error_data_result(message="`name` is not empty.")
-        if req["name"].lower() != res["name"].lower() \
-                and len(
-            DialogService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) > 0:
-            return get_error_data_result(message="Duplicated chat name in updating dataset.")
+            return get_error_data_result(message="`name` cannot be empty.")
+        if req["name"].lower() != res["name"].lower() and len(DialogService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) > 0:
+            return get_error_data_result(message="Duplicated chat name in updating chat.")
     if "prompt_config" in req:
         res["prompt_config"].update(req["prompt_config"])
         for p in res["prompt_config"]["parameters"]:
@@ -235,36 +223,58 @@ def update(tenant_id,chat_id):
     return get_result()
 
 
-@manager.route('/chats', methods=['DELETE'])
+@manager.route("/chats", methods=["DELETE"])  # noqa: F821
 @token_required
 def delete(tenant_id):
+    errors = []
+    success_count = 0
     req = request.json
     if not req:
-        ids=None
+        ids = None
     else:
-        ids=req.get("ids")
+        ids = req.get("ids")
     if not ids:
         id_list = []
-        dias=DialogService.query(tenant_id=tenant_id,status=StatusEnum.VALID.value)
+        dias = DialogService.query(tenant_id=tenant_id, status=StatusEnum.VALID.value)
         for dia in dias:
             id_list.append(dia.id)
     else:
-        id_list=ids
-    for id in id_list:
+        id_list = ids
+
+    unique_id_list, duplicate_messages = check_duplicate_ids(id_list, "assistant")
+
+    for id in unique_id_list:
         if not DialogService.query(tenant_id=tenant_id, id=id, status=StatusEnum.VALID.value):
-            return get_error_data_result(message=f"You don't own the chat {id}")
+            errors.append(f"Assistant({id}) not found.")
+            continue
         temp_dict = {"status": StatusEnum.INVALID.value}
         DialogService.update_by_id(id, temp_dict)
+        success_count += 1
+
+    if errors:
+        if success_count > 0:
+            return get_result(data={"success_count": success_count, "errors": errors}, message=f"Partially deleted {success_count} chats with {len(errors)} errors")
+        else:
+            return get_error_data_result(message="; ".join(errors))
+
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(message=f"Partially deleted {success_count} chats with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages})
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+
     return get_result()
 
-@manager.route('/chats', methods=['GET'])
+
+@manager.route("/chats", methods=["GET"])  # noqa: F821
 @token_required
 def list_chat(tenant_id):
     id = request.args.get("id")
     name = request.args.get("name")
-    chat = DialogService.query(id=id,name=name,status=StatusEnum.VALID.value,tenant_id=tenant_id)
-    if not chat:
-        return get_error_data_result(message="The chat doesn't exist")
+    if id or name:
+        chat = DialogService.query(id=id, name=name, status=StatusEnum.VALID.value, tenant_id=tenant_id)
+        if not chat:
+            return get_error_data_result(message="The chat doesn't exist")
     page_number = int(request.args.get("page", 1))
     items_per_page = int(request.args.get("page_size", 30))
     orderby = request.args.get("orderby", "create_time")
@@ -272,29 +282,28 @@ def list_chat(tenant_id):
         desc = False
     else:
         desc = True
-    chats = DialogService.get_list(tenant_id,page_number,items_per_page,orderby,desc,id,name)
+    chats = DialogService.get_list(tenant_id, page_number, items_per_page, orderby, desc, id, name)
     if not chats:
         return get_result(data=[])
     list_assts = []
-    renamed_dict = {}
-    key_mapping = {"parameters": "variables",
-                   "prologue": "opener",
-                   "quote": "show_quote",
-                   "system": "prompt",
-                   "rerank_id": "rerank_model",
-                   "vector_similarity_weight": "keywords_similarity_weight",
-                   "do_refer":"show_quotation"}
+    key_mapping = {
+        "parameters": "variables",
+        "prologue": "opener",
+        "quote": "show_quote",
+        "system": "prompt",
+        "rerank_id": "rerank_model",
+        "vector_similarity_weight": "keywords_similarity_weight",
+        "do_refer": "show_quotation",
+    }
     key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
     for res in chats:
+        renamed_dict = {}
         for key, value in res["prompt_config"].items():
             new_key = key_mapping.get(key, key)
             renamed_dict[new_key] = value
         res["prompt"] = renamed_dict
         del res["prompt_config"]
-        new_dict = {"similarity_threshold": res["similarity_threshold"],
-                    "keywords_similarity_weight": res["vector_similarity_weight"],
-                    "top_n": res["top_n"],
-                    "rerank_model": res['rerank_id']}
+        new_dict = {"similarity_threshold": res["similarity_threshold"], "keywords_similarity_weight": 1 - res["vector_similarity_weight"], "top_n": res["top_n"], "rerank_model": res["rerank_id"]}
         res["prompt"].update(new_dict)
         for key in key_list:
             del res[key]
@@ -303,11 +312,12 @@ def list_chat(tenant_id):
         kb_list = []
         for kb_id in res["kb_ids"]:
             kb = KnowledgebaseService.query(id=kb_id)
-            if not kb :
-                return get_error_data_result(message=f"Don't exist the kb {kb_id}")
+            if not kb:
+                logging.warning(f"The kb {kb_id} does not exist.")
+                continue
             kb_list.append(kb[0].to_json())
         del res["kb_ids"]
         res["datasets"] = kb_list
         res["avatar"] = res.pop("icon")
         list_assts.append(res)
-    return get_result(data=list_assts)
+    return get_result(data=list_assts)

api/apps/sdk/dataset.py

@@ -14,27 +14,43 @@
 #  limitations under the License.
 #
 
+
+import logging
+
 from flask import request
-from api.db import StatusEnum, FileSource
+from peewee import OperationalError
+
+from api.db import FileSource, StatusEnum
 from api.db.db_models import File
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.llm_service import TenantLLMService, LLMService
 from api.db.services.user_service import TenantService
-from api import settings
 from api.utils import get_uuid
 from api.utils.api_utils import (
-    get_result,
-    token_required,
+    deep_merge,
+    get_error_argument_result,
     get_error_data_result,
-    valid,
+    get_error_operating_result,
+    get_error_permission_result,
     get_parser_config,
+    get_result,
+    remap_dictionary_keys,
+    token_required,
+    verify_embedding_availability,
+)
+from api.utils.validation_utils import (
+    CreateDatasetReq,
+    DeleteDatasetReq,
+    ListDatasetReq,
+    UpdateDatasetReq,
+    validate_and_parse_json_request,
+    validate_and_parse_request_args,
 )
 
 
-@manager.route("/datasets", methods=["POST"])
+@manager.route("/datasets", methods=["POST"])  # noqa: F821
 @token_required
 def create(tenant_id):
     """
@@ -62,19 +78,28 @@ def create(tenant_id):
             name:
               type: string
               description: Name of the dataset.
+            avatar:
+              type: string
+              description: Base64 encoding of the avatar.
+            description:
+              type: string
+              description: Description of the dataset.
+            embedding_model:
+              type: string
+              description: Embedding model Name.
             permission:
               type: string
               enum: ['me', 'team']
               description: Dataset permission.
-            language:
-              type: string
-              enum: ['Chinese', 'English']
-              description: Language of the dataset.
             chunk_method:
               type: string
-              enum: ["naive", "manual", "qa", "table", "paper", "book", "laws",
-                     "presentation", "picture", "one", "knowledge_graph", "email"]
+              enum: ["naive", "book", "email", "laws", "manual", "one", "paper",
+                     "picture", "presentation", "qa", "table", "tag"
+                     ]
               description: Chunking method.
+            pagerank:
+              type: integer
+              description: Set page rank.
             parser_config:
               type: object
               description: Parser configuration.
@@ -87,110 +112,62 @@ def create(tenant_id):
             data:
               type: object
     """
-    req = request.json
-    e, t = TenantService.get_by_id(tenant_id)
-    permission = req.get("permission")
-    language = req.get("language")
-    chunk_method = req.get("chunk_method")
-    parser_config = req.get("parser_config")
-    valid_permission = ["me", "team"]
-    valid_language = ["Chinese", "English"]
-    valid_chunk_method = [
-        "naive",
-        "manual",
-        "qa",
-        "table",
-        "paper",
-        "book",
-        "laws",
-        "presentation",
-        "picture",
-        "one",
-        "knowledge_graph",
-        "email",
-    ]
-    check_validation = valid(
-        permission,
-        valid_permission,
-        language,
-        valid_language,
-        chunk_method,
-        valid_chunk_method,
-    )
-    if check_validation:
-        return check_validation
-    req["parser_config"] = get_parser_config(chunk_method, parser_config)
-    if "tenant_id" in req:
-        return get_error_data_result(message="`tenant_id` must not be provided")
-    if "chunk_count" in req or "document_count" in req:
-        return get_error_data_result(
-            message="`chunk_count` or `document_count` must not be provided"
-        )
-    if "name" not in req:
-        return get_error_data_result(message="`name` is not empty!")
+    # Field name transformations during model dump:
+    # | Original       | Dump Output  |
+    # |----------------|-------------|
+    # | embedding_model| embd_id     |
+    # | chunk_method   | parser_id   |
+    req, err = validate_and_parse_json_request(request, CreateDatasetReq)
+    if err is not None:
+        return get_error_argument_result(err)
+
+    try:
+        if KnowledgebaseService.get_or_none(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
+            return get_error_operating_result(message=f"Dataset name '{req['name']}' already exists")
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
+
+    req["parser_config"] = get_parser_config(req["parser_id"], req["parser_config"])
     req["id"] = get_uuid()
-    req["name"] = req["name"].strip()
-    if req["name"] == "":
-        return get_error_data_result(message="`name` is not empty string!")
-    if KnowledgebaseService.query(
-        name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value
-    ):
-        return get_error_data_result(
-            message="Duplicated dataset name in creating dataset."
-        )
-    req["tenant_id"] = req["created_by"] = tenant_id
-    if not req.get("embedding_model"):
-        req["embedding_model"] = t.embd_id
+    req["tenant_id"] = tenant_id
+    req["created_by"] = tenant_id
+
+    try:
+        ok, t = TenantService.get_by_id(tenant_id)
+        if not ok:
+            return get_error_permission_result(message="Tenant not found")
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
+
+    if not req.get("embd_id"):
+        req["embd_id"] = t.embd_id
     else:
-        valid_embedding_models = [
-            "BAAI/bge-large-zh-v1.5",
-            "BAAI/bge-base-en-v1.5",
-            "BAAI/bge-large-en-v1.5",
-            "BAAI/bge-small-en-v1.5",
-            "BAAI/bge-small-zh-v1.5",
-            "jinaai/jina-embeddings-v2-base-en",
-            "jinaai/jina-embeddings-v2-small-en",
-            "nomic-ai/nomic-embed-text-v1.5",
-            "sentence-transformers/all-MiniLM-L6-v2",
-            "text-embedding-v2",
-            "text-embedding-v3",
-            "maidalun1020/bce-embedding-base_v1",
-        ]
-        embd_model = LLMService.query(
-            llm_name=req["embedding_model"], model_type="embedding"
-        )
-        if embd_model:
-            if req["embedding_model"] not in valid_embedding_models and not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding",llm_name=req.get("embedding_model"),):
-                return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
-        if not embd_model:
-            embd_model=TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model"))
-        if not embd_model:
-            return get_error_data_result(
-                f"`embedding_model` {req.get('embedding_model')} doesn't exist"
-            )
-    key_mapping = {
-        "chunk_num": "chunk_count",
-        "doc_num": "document_count",
-        "parser_id": "chunk_method",
-        "embd_id": "embedding_model",
-    }
-    mapped_keys = {
-        new_key: req[old_key]
-        for new_key, old_key in key_mapping.items()
-        if old_key in req
-    }
-    req.update(mapped_keys)
-    if not KnowledgebaseService.save(**req):
-        return get_error_data_result(message="Create dataset error.(Database error)")
-    renamed_data = {}
-    e, k = KnowledgebaseService.get_by_id(req["id"])
-    for key, value in k.to_dict().items():
-        new_key = key_mapping.get(key, key)
-        renamed_data[new_key] = value
-    return get_result(data=renamed_data)
+        ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
+        if not ok:
+            return err
+
+    try:
+        if not KnowledgebaseService.save(**req):
+            return get_error_data_result(message="Create dataset error.(Database error)")
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
+
+    try:
+        ok, k = KnowledgebaseService.get_by_id(req["id"])
+        if not ok:
+            return get_error_data_result(message="Dataset created failed")
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
+
+    response_data = remap_dictionary_keys(k.to_dict())
+    return get_result(data=response_data)
 
 
-@manager.route("/datasets", methods=["DELETE"])
+@manager.route("/datasets", methods=["DELETE"])  # noqa: F821
 @token_required
 def delete(tenant_id):
     """
@@ -212,55 +189,88 @@ def delete(tenant_id):
         required: true
         schema:
           type: object
+          required:
+            - ids
           properties:
             ids:
-              type: array
+              type: array or null
               items:
                 type: string
-              description: List of dataset IDs to delete.
+              description: |
+                Specifies the datasets to delete:
+                - If `null`, all datasets will be deleted.
+                - If an array of IDs, only the specified datasets will be deleted.
+                - If an empty array, no datasets will be deleted.
     responses:
       200:
         description: Successful operation.
         schema:
           type: object
     """
-    req = request.json
-    if not req:
-        ids = None
+    req, err = validate_and_parse_json_request(request, DeleteDatasetReq)
+    if err is not None:
+        return get_error_argument_result(err)
+
+    kb_id_instance_pairs = []
+    if req["ids"] is None:
+        try:
+            kbs = KnowledgebaseService.query(tenant_id=tenant_id)
+            for kb in kbs:
+                kb_id_instance_pairs.append((kb.id, kb))
+        except OperationalError as e:
+            logging.exception(e)
+            return get_error_data_result(message="Database operation failed")
     else:
-        ids = req.get("ids")
-    if not ids:
-        id_list = []
-        kbs = KnowledgebaseService.query(tenant_id=tenant_id)
-        for kb in kbs:
-            id_list.append(kb.id)
-    else:
-        id_list = ids
-    for id in id_list:
-        kbs = KnowledgebaseService.query(id=id, tenant_id=tenant_id)
-        if not kbs:
-            return get_error_data_result(message=f"You don't own the dataset {id}")
-        for doc in DocumentService.query(kb_id=id):
-            if not DocumentService.remove_document(doc, tenant_id):
-                return get_error_data_result(
-                    message="Remove document error.(Database error)"
+        error_kb_ids = []
+        for kb_id in req["ids"]:
+            try:
+                kb = KnowledgebaseService.get_or_none(id=kb_id, tenant_id=tenant_id)
+                if kb is None:
+                    error_kb_ids.append(kb_id)
+                    continue
+                kb_id_instance_pairs.append((kb_id, kb))
+            except OperationalError as e:
+                logging.exception(e)
+                return get_error_data_result(message="Database operation failed")
+        if len(error_kb_ids) > 0:
+            return get_error_permission_result(message=f"""User '{tenant_id}' lacks permission for datasets: '{", ".join(error_kb_ids)}'""")
+
+    errors = []
+    success_count = 0
+    for kb_id, kb in kb_id_instance_pairs:
+        try:
+            for doc in DocumentService.query(kb_id=kb_id):
+                if not DocumentService.remove_document(doc, tenant_id):
+                    errors.append(f"Remove document '{doc.id}' error for dataset '{kb_id}'")
+                    continue
+                f2d = File2DocumentService.get_by_document_id(doc.id)
+                FileService.filter_delete(
+                    [
+                        File.source_type == FileSource.KNOWLEDGEBASE,
+                        File.id == f2d[0].file_id,
+                    ]
                 )
-            f2d = File2DocumentService.get_by_document_id(doc.id)
-            FileService.filter_delete(
-                [
-                    File.source_type == FileSource.KNOWLEDGEBASE,
-                    File.id == f2d[0].file_id,
-                ]
-            )
-            FileService.filter_delete(
-                [File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
-            File2DocumentService.delete_by_document_id(doc.id)
-        if not KnowledgebaseService.delete_by_id(id):
-            return get_error_data_result(message="Delete dataset error.(Database error)")
-    return get_result(code=settings.RetCode.SUCCESS)
+                File2DocumentService.delete_by_document_id(doc.id)
+            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kb.name])
+            if not KnowledgebaseService.delete_by_id(kb_id):
+                errors.append(f"Delete dataset error for {kb_id}")
+                continue
+            success_count += 1
+        except OperationalError as e:
+            logging.exception(e)
+            return get_error_data_result(message="Database operation failed")
+
+    if not errors:
+        return get_result()
+
+    error_message = f"Successfully deleted {success_count} datasets, {len(errors)} failed. Details: {'; '.join(errors)[:128]}..."
+    if success_count == 0:
+        return get_error_data_result(message=error_message)
+
+    return get_result(data={"success_count": success_count, "errors": errors[:5]}, message=error_message)
 
 
-@manager.route("/datasets/<dataset_id>", methods=["PUT"])
+@manager.route("/datasets/<dataset_id>", methods=["PUT"])  # noqa: F821
 @token_required
 def update(tenant_id, dataset_id):
     """
@@ -291,19 +301,28 @@ def update(tenant_id, dataset_id):
             name:
               type: string
               description: New name of the dataset.
+            avatar:
+              type: string
+              description: Updated base64 encoding of the avatar.
+            description:
+              type: string
+              description: Updated description of the dataset.
+            embedding_model:
+              type: string
+              description: Updated embedding model Name.
             permission:
               type: string
               enum: ['me', 'team']
-              description: Updated permission.
-            language:
-              type: string
-              enum: ['Chinese', 'English']
-              description: Updated language.
+              description: Updated dataset permission.
             chunk_method:
               type: string
-              enum: ["naive", "manual", "qa", "table", "paper", "book", "laws",
-                     "presentation", "picture", "one", "knowledge_graph", "email"]
+              enum: ["naive", "book", "email", "laws", "manual", "one", "paper",
+                     "picture", "presentation", "qa", "table", "tag"
+                     ]
               description: Updated chunking method.
+            pagerank:
+              type: integer
+              description: Updated page rank.
             parser_config:
               type: object
               description: Updated parser configuration.
@@ -313,125 +332,65 @@ def update(tenant_id, dataset_id):
         schema:
           type: object
     """
-    if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
-        return get_error_data_result(message="You don't own the dataset")
-    req = request.json
-    e, t = TenantService.get_by_id(tenant_id)
-    invalid_keys = {"id", "embd_id", "chunk_num", "doc_num", "parser_id"}
-    if any(key in req for key in invalid_keys):
-        return get_error_data_result(message="The input parameters are invalid.")
-    permission = req.get("permission")
-    language = req.get("language")
-    chunk_method = req.get("chunk_method")
-    parser_config = req.get("parser_config")
-    valid_permission = ["me", "team"]
-    valid_language = ["Chinese", "English"]
-    valid_chunk_method = [
-        "naive",
-        "manual",
-        "qa",
-        "table",
-        "paper",
-        "book",
-        "laws",
-        "presentation",
-        "picture",
-        "one",
-        "knowledge_graph",
-        "email",
-    ]
-    check_validation = valid(
-        permission,
-        valid_permission,
-        language,
-        valid_language,
-        chunk_method,
-        valid_chunk_method,
-    )
-    if check_validation:
-        return check_validation
-    if "tenant_id" in req:
-        if req["tenant_id"] != tenant_id:
-            return get_error_data_result(message="Can't change `tenant_id`.")
-    e, kb = KnowledgebaseService.get_by_id(dataset_id)
-    if "parser_config" in req:
-        temp_dict = kb.parser_config
-        temp_dict.update(req["parser_config"])
-        req["parser_config"] = temp_dict
-    if "chunk_count" in req:
-        if req["chunk_count"] != kb.chunk_num:
-            return get_error_data_result(message="Can't change `chunk_count`.")
-        req.pop("chunk_count")
-    if "document_count" in req:
-        if req["document_count"] != kb.doc_num:
-            return get_error_data_result(message="Can't change `document_count`.")
-        req.pop("document_count")
-    if "chunk_method" in req:
-        if kb.chunk_num != 0 and req["chunk_method"] != kb.parser_id:
-            return get_error_data_result(
-                message="If `chunk_count` is not 0, `chunk_method` is not changeable."
-            )
-        req["parser_id"] = req.pop("chunk_method")
-        if req["parser_id"] != kb.parser_id:
-            if not req.get("parser_config"):
-                req["parser_config"] = get_parser_config(chunk_method, parser_config)
-    if "embedding_model" in req:
-        if kb.chunk_num != 0 and req["embedding_model"] != kb.embd_id:
-            return get_error_data_result(
-                message="If `chunk_count` is not 0, `embedding_model` is not changeable."
-            )
-        if not req.get("embedding_model"):
-            return get_error_data_result("`embedding_model` can't be empty")
-        valid_embedding_models = [
-            "BAAI/bge-large-zh-v1.5",
-            "BAAI/bge-base-en-v1.5",
-            "BAAI/bge-large-en-v1.5",
-            "BAAI/bge-small-en-v1.5",
-            "BAAI/bge-small-zh-v1.5",
-            "jinaai/jina-embeddings-v2-base-en",
-            "jinaai/jina-embeddings-v2-small-en",
-            "nomic-ai/nomic-embed-text-v1.5",
-            "sentence-transformers/all-MiniLM-L6-v2",
-            "text-embedding-v2",
-            "text-embedding-v3",
-            "maidalun1020/bce-embedding-base_v1",
-        ]
-        embd_model = LLMService.query(
-            llm_name=req["embedding_model"], model_type="embedding"
-        )
-        if embd_model:
-            if req["embedding_model"] not in valid_embedding_models and not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding",llm_name=req.get("embedding_model"),):
-                return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
-        if not embd_model:
-            embd_model=TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model"))
+    # Field name transformations during model dump:
+    # | Original       | Dump Output  |
+    # |----------------|-------------|
+    # | embedding_model| embd_id     |
+    # | chunk_method   | parser_id   |
+    extras = {"dataset_id": dataset_id}
+    req, err = validate_and_parse_json_request(request, UpdateDatasetReq, extras=extras, exclude_unset=True)
+    if err is not None:
+        return get_error_argument_result(err)
 
-        if not embd_model:
-            return get_error_data_result(
-                f"`embedding_model` {req.get('embedding_model')} doesn't exist"
-            )
-        req["embd_id"] = req.pop("embedding_model")
-    if "name" in req:
-        req["name"] = req["name"].strip()
-        if (
-            req["name"].lower() != kb.name.lower()
-            and len(
-                KnowledgebaseService.query(
-                    name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value
-                )
-            )
-            > 0
-        ):
-            return get_error_data_result(
-                message="Duplicated dataset name in updating dataset."
-            )
-    if not KnowledgebaseService.update_by_id(kb.id, req):
-        return get_error_data_result(message="Update dataset error.(Database error)")
-    return get_result(code=settings.RetCode.SUCCESS)
+    if not req:
+        return get_error_argument_result(message="No properties were modified")
+
+    try:
+        kb = KnowledgebaseService.get_or_none(id=dataset_id, tenant_id=tenant_id)
+        if kb is None:
+            return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{dataset_id}'")
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
+
+    if req.get("parser_config"):
+        req["parser_config"] = deep_merge(kb.parser_config, req["parser_config"])
+
+    if (chunk_method := req.get("parser_id")) and chunk_method != kb.parser_id:
+        if not req.get("parser_config"):
+            req["parser_config"] = get_parser_config(chunk_method, None)
+    elif "parser_config" in req and not req["parser_config"]:
+        del req["parser_config"]
+
+    if "name" in req and req["name"].lower() != kb.name.lower():
+        try:
+            exists = KnowledgebaseService.get_or_none(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)
+            if exists:
+                return get_error_data_result(message=f"Dataset name '{req['name']}' already exists")
+        except OperationalError as e:
+            logging.exception(e)
+            return get_error_data_result(message="Database operation failed")
+
+    if "embd_id" in req:
+        if kb.chunk_num != 0 and req["embd_id"] != kb.embd_id:
+            return get_error_data_result(message=f"When chunk_num ({kb.chunk_num}) > 0, embedding_model must remain {kb.embd_id}")
+        ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
+        if not ok:
+            return err
+
+    try:
+        if not KnowledgebaseService.update_by_id(kb.id, req):
+            return get_error_data_result(message="Update dataset error.(Database error)")
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
+
+    return get_result()
 
 
-@manager.route("/datasets", methods=["GET"])
+@manager.route("/datasets", methods=["GET"])  # noqa: F821
 @token_required
-def list(tenant_id):
+def list_datasets(tenant_id):
     """
     List datasets.
     ---
@@ -460,7 +419,7 @@ def list(tenant_id):
         name: page_size
         type: integer
         required: false
-        default: 1024
+        default: 30
         description: Number of items per page.
       - in: query
         name: orderby
@@ -487,45 +446,46 @@ def list(tenant_id):
           items:
             type: object
     """
-    id = request.args.get("id")
-    name = request.args.get("name")
-    if id:
-        kbs = KnowledgebaseService.get_kb_by_id(id,tenant_id)
+    args, err = validate_and_parse_request_args(request, ListDatasetReq)
+    if err is not None:
+        return get_error_argument_result(err)
+
+    kb_id = request.args.get("id")
+    name = args.get("name")
+    if kb_id:
+        try:
+            kbs = KnowledgebaseService.get_kb_by_id(kb_id, tenant_id)
+        except OperationalError as e:
+            logging.exception(e)
+            return get_error_data_result(message="Database operation failed")
         if not kbs:
-            return get_error_data_result(f"You don't own the dataset {id}")
+            return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{kb_id}'")
     if name:
-        kbs = KnowledgebaseService.get_kb_by_name(name,tenant_id)
+        try:
+            kbs = KnowledgebaseService.get_kb_by_name(name, tenant_id)
+        except OperationalError as e:
+            logging.exception(e)
+            return get_error_data_result(message="Database operation failed")
         if not kbs:
-            return get_error_data_result(f"You don't own the dataset {name}")
-    page_number = int(request.args.get("page", 1))
-    items_per_page = int(request.args.get("page_size", 30))
-    orderby = request.args.get("orderby", "create_time")
-    if request.args.get("desc") == "False" or request.args.get("desc") == "false":
-        desc = False
-    else:
-        desc = True
-    tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)
-    kbs = KnowledgebaseService.get_list(
-        [m["tenant_id"] for m in tenants],
-        tenant_id,
-        page_number,
-        items_per_page,
-        orderby,
-        desc,
-        id,
-        name,
-    )
-    renamed_list = []
+            return get_error_permission_result(message=f"User '{tenant_id}' lacks permission for dataset '{name}'")
+
+    try:
+        tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)
+        kbs = KnowledgebaseService.get_list(
+            [m["tenant_id"] for m in tenants],
+            tenant_id,
+            args["page"],
+            args["page_size"],
+            args["orderby"],
+            args["desc"],
+            kb_id,
+            name,
+        )
+    except OperationalError as e:
+        logging.exception(e)
+        return get_error_data_result(message="Database operation failed")
+
+    response_data_list = []
     for kb in kbs:
-        key_mapping = {
-            "chunk_num": "chunk_count",
-            "doc_num": "document_count",
-            "parser_id": "chunk_method",
-            "embd_id": "embedding_model",
-        }
-        renamed_data = {}
-        for key, value in kb.items():
-            new_key = key_mapping.get(key, key)
-            renamed_data[new_key] = value
-        renamed_list.append(renamed_data)
-    return get_result(data=renamed_list)
+        response_data_list.append(remap_dictionary_keys(kb))
+    return get_result(data=response_data_list)

api/apps/sdk/dify_retrieval.py

@@ -15,20 +15,22 @@
 #
 from flask import request, jsonify
 
-from api.db import LLMType, ParserType
+from api.db import LLMType
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api import settings
 from api.utils.api_utils import validate_request, build_error_result, apikey_required
+from rag.app.tag import label_question
 
 
-@manager.route('/dify/retrieval', methods=['POST'])
+@manager.route('/dify/retrieval', methods=['POST'])  # noqa: F821
 @apikey_required
 @validate_request("knowledge_id", "query")
 def retrieval(tenant_id):
     req = request.json
     question = req["query"]
     kb_id = req["knowledge_id"]
+    use_kg = req.get("use_kg", False)
     retrieval_setting = req.get("retrieval_setting", {})
     similarity_threshold = float(retrieval_setting.get("score_threshold", 0.0))
     top = int(retrieval_setting.get("top_k", 1024))
@@ -44,8 +46,7 @@ def retrieval(tenant_id):
 
         embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
 
-        retr = settings.retrievaler if kb.parser_id != ParserType.KG else settings.kg_retrievaler
-        ranks = retr.retrieval(
+        ranks = settings.retrievaler.retrieval(
             question,
             embd_mdl,
             kb.tenant_id,
@@ -54,13 +55,24 @@ def retrieval(tenant_id):
             page_size=top,
             similarity_threshold=similarity_threshold,
             vector_similarity_weight=0.3,
-            top=top
+            top=top,
+            rank_feature=label_question(question, [kb])
         )
+
+        if use_kg:
+            ck = settings.kg_retrievaler.retrieval(question,
+                                                   [tenant_id],
+                                                   [kb_id],
+                                                   embd_mdl,
+                                                   LLMBundle(kb.tenant_id, LLMType.CHAT))
+            if ck["content_with_weight"]:
+                ranks["chunks"].insert(0, ck)
+
         records = []
         for c in ranks["chunks"]:
             c.pop("vector", None)
             records.append({
-                "content": c["content_ltks"],
+                "content": c["content_with_weight"],
                 "score": c["similarity"],
                 "title": c["docnm_kwd"],
                 "metadata": {}

api/apps/sdk/doc.py

@@ -16,13 +16,12 @@
 import pathlib
 import datetime
 
-from api.db.services.dialog_service import keyword_extraction
 from rag.app.qa import rmPrefix, beAdoc
 from rag.nlp import rag_tokenizer
 from api.db import LLMType, ParserType
-from api.db.services.llm_service import TenantLLMService
+from api.db.services.llm_service import TenantLLMService, LLMBundle
 from api import settings
-import hashlib
+import xxhash
 import re
 from api.utils.api_utils import token_required
 from api.db.db_models import Task
@@ -37,16 +36,39 @@ from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.utils.api_utils import construct_json_result, get_parser_config
+from api.utils.api_utils import construct_json_result, get_parser_config, check_duplicate_ids
 from rag.nlp import search
+from rag.prompts import keyword_extraction
+from rag.app.tag import label_question
 from rag.utils import rmSpace
 from rag.utils.storage_factory import STORAGE_IMPL
-import os
+
+from pydantic import BaseModel, Field, validator
 
 MAXIMUM_OF_UPLOADING_FILES = 256
 
 
-@manager.route("/datasets/<dataset_id>/documents", methods=["POST"])
+class Chunk(BaseModel):
+    id: str = ""
+    content: str = ""
+    document_id: str = ""
+    docnm_kwd: str = ""
+    important_keywords: list = Field(default_factory=list)
+    questions: list = Field(default_factory=list)
+    question_tks: str = ""
+    image_id: str = ""
+    available: bool = True
+    positions: list[list[int]] = Field(default_factory=list)
+
+    @validator('positions')
+    def validate_positions(cls, value):
+        for sublist in value:
+            if len(sublist) != 5:
+                raise ValueError("Each sublist in positions must have a length of 5")
+        return value
+
+
+@manager.route("/datasets/<dataset_id>/documents", methods=["POST"])  # noqa: F821
 @token_required
 def upload(dataset_id, tenant_id):
     """
@@ -115,6 +137,10 @@ def upload(dataset_id, tenant_id):
             return get_result(
                 message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR
             )
+        if len(file_obj.filename.encode("utf-8")) >= 128:
+            return get_result(
+                message="File name should be less than 128 bytes.", code=settings.RetCode.ARGUMENT_ERROR
+            )
     '''
     # total size
     total_size = 0
@@ -154,7 +180,7 @@ def upload(dataset_id, tenant_id):
     return get_result(data=renamed_doc_list)
 
 
-@manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["PUT"])
+@manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["PUT"])  # noqa: F821
 @token_required
 def update_doc(tenant_id, dataset_id, document_id):
     """
@@ -196,6 +222,9 @@ def update_doc(tenant_id, dataset_id, document_id):
             chunk_method:
               type: string
               description: Chunking method.
+            enabled:
+              type: boolean
+              description: Document status.
     responses:
       200:
         description: Document updated successfully.
@@ -205,6 +234,10 @@ def update_doc(tenant_id, dataset_id, document_id):
     req = request.json
     if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
         return get_error_data_result(message="You don't own the dataset.")
+    e, kb = KnowledgebaseService.get_by_id(dataset_id)
+    if not e:
+        return get_error_data_result(
+            message="Can't find this knowledgebase!")
     doc = DocumentService.query(kb_id=dataset_id, id=document_id)
     if not doc:
         return get_error_data_result(message="The dataset doesn't own the document.")
@@ -219,7 +252,17 @@ def update_doc(tenant_id, dataset_id, document_id):
         if req["progress"] != doc.progress:
             return get_error_data_result(message="Can't change `progress`.")
 
+    if "meta_fields" in req:
+        if not isinstance(req["meta_fields"], dict):
+            return get_error_data_result(message="meta_fields must be a dictionary")
+        DocumentService.update_meta_fields(document_id, req["meta_fields"])
+
     if "name" in req and req["name"] != doc.name:
+        if len(req["name"].encode("utf-8")) >= 128:
+            return get_result(
+                message="The name should be less than 128 bytes.",
+                code=settings.RetCode.ARGUMENT_ERROR,
+            )
         if (
                 pathlib.Path(req["name"].lower()).suffix
                 != pathlib.Path(doc.name.lower()).suffix
@@ -240,6 +283,7 @@ def update_doc(tenant_id, dataset_id, document_id):
         if informs:
             e, file = FileService.get_by_id(informs[0].file_id)
             FileService.update_by_id(file.id, {"name": req["name"]})
+
     if "parser_config" in req:
         DocumentService.update_parser_config(doc.id, req["parser_config"])
     if "chunk_method" in req:
@@ -256,6 +300,7 @@ def update_doc(tenant_id, dataset_id, document_id):
             "one",
             "knowledge_graph",
             "email",
+            "tag"
         }
         if req.get("chunk_method") not in valid_chunk_method:
             return get_error_data_result(
@@ -294,10 +339,26 @@ def update_doc(tenant_id, dataset_id, document_id):
                 return get_error_data_result(message="Document not found!")
             settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), dataset_id)
 
+    if "enabled" in req:
+        status = int(req["enabled"])
+        if doc.status != req["enabled"]:
+            try:
+                if not DocumentService.update_by_id(
+                        doc.id, {"status": str(status)}):
+                    return get_error_data_result(
+                        message="Database error (Document update)!")
+
+                settings.docStoreConn.update({"doc_id": doc.id}, {"available_int": status},
+                                             search.index_name(kb.tenant_id), doc.kb_id)
+                return get_result(data=True)
+            except Exception as e:
+                return server_error_response(e)
+
     return get_result()
 
 
-@manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["GET"])
+
+@manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["GET"])  # noqa: F821
 @token_required
 def download(tenant_id, dataset_id, document_id):
     """
@@ -335,6 +396,10 @@ def download(tenant_id, dataset_id, document_id):
         schema:
           type: object
     """
+    if not document_id:
+        return get_error_data_result(
+            message="Specify document_id please."
+        )
     if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
         return get_error_data_result(message=f"You do not own the dataset {dataset_id}.")
     doc = DocumentService.query(kb_id=dataset_id, id=document_id)
@@ -361,7 +426,7 @@ def download(tenant_id, dataset_id, document_id):
     )
 
 
-@manager.route("/datasets/<dataset_id>/documents", methods=["GET"])
+@manager.route("/datasets/<dataset_id>/documents", methods=["GET"])  # noqa: F821
 @token_required
 def list_docs(dataset_id, tenant_id):
     """
@@ -451,10 +516,12 @@ def list_docs(dataset_id, tenant_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}. ")
     id = request.args.get("id")
     name = request.args.get("name")
-    if not DocumentService.query(id=id, kb_id=dataset_id):
+
+    if id and not DocumentService.query(id=id, kb_id=dataset_id):
         return get_error_data_result(message=f"You don't own the document {id}.")
-    if not DocumentService.query(name=name, kb_id=dataset_id):
+    if name and not DocumentService.query(name=name, kb_id=dataset_id):
         return get_error_data_result(message=f"You don't own the document {name}.")
+
     page = int(request.args.get("page", 1))
     keywords = request.args.get("keywords", "")
     page_size = int(request.args.get("page_size", 30))
@@ -495,7 +562,7 @@ def list_docs(dataset_id, tenant_id):
     return get_result(data={"total": tol, "docs": renamed_doc_list})
 
 
-@manager.route("/datasets/<dataset_id>/documents", methods=["DELETE"])
+@manager.route("/datasets/<dataset_id>/documents", methods=["DELETE"])  # noqa: F821
 @token_required
 def delete(tenant_id, dataset_id):
     """
@@ -548,15 +615,22 @@ def delete(tenant_id, dataset_id):
             doc_list.append(doc.id)
     else:
         doc_list = doc_ids
+
+    unique_doc_ids, duplicate_messages = check_duplicate_ids(doc_list, "document")
+    doc_list = unique_doc_ids
+
     root_folder = FileService.get_root_folder(tenant_id)
     pf_id = root_folder["id"]
     FileService.init_knowledgebase_docs(pf_id, tenant_id)
     errors = ""
+    not_found = []
+    success_count = 0
     for doc_id in doc_list:
         try:
             e, doc = DocumentService.get_by_id(doc_id)
             if not e:
-                return get_error_data_result(message="Document not found!")
+                not_found.append(doc_id)
+                continue
             tenant_id = DocumentService.get_tenant_id(doc_id)
             if not tenant_id:
                 return get_error_data_result(message="Tenant not found!")
@@ -578,16 +652,26 @@ def delete(tenant_id, dataset_id):
             File2DocumentService.delete_by_document_id(doc_id)
 
             STORAGE_IMPL.rm(b, n)
+            success_count += 1
         except Exception as e:
             errors += str(e)
 
+    if not_found:
+        return get_result(message=f"Documents not found: {not_found}", code=settings.RetCode.DATA_ERROR)
+
     if errors:
         return get_result(message=errors, code=settings.RetCode.SERVER_ERROR)
 
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(message=f"Partially deleted {success_count} datasets with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages},)
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+
     return get_result()
 
 
-@manager.route("/datasets/<dataset_id>/chunks", methods=["POST"])
+@manager.route("/datasets/<dataset_id>/chunks", methods=["POST"])  # noqa: F821
 @token_required
 def parse(tenant_id, dataset_id):
     """
@@ -631,18 +715,24 @@ def parse(tenant_id, dataset_id):
     req = request.json
     if not req.get("document_ids"):
         return get_error_data_result("`document_ids` is required")
-    for id in req["document_ids"]:
+    doc_list = req.get("document_ids")
+    unique_doc_ids, duplicate_messages = check_duplicate_ids(doc_list, "document")
+    doc_list = unique_doc_ids
+
+    not_found = []
+    success_count = 0
+    for id in doc_list:
         doc = DocumentService.query(id=id, kb_id=dataset_id)
+        if not doc:
+            not_found.append(id)
+            continue
         if not doc:
             return get_error_data_result(message=f"You don't own the document {id}.")
-        if doc[0].progress != 0.0:
+        if 0.0 < doc[0].progress < 1.0:
             return get_error_data_result(
-                "Can't stop parsing document with progress at 0 or 100"
+                "Can't parse document that is currently being processed"
             )
-        info = {"run": "1", "progress": 0}
-        info["progress_msg"] = ""
-        info["chunk_num"] = 0
-        info["token_num"] = 0
+        info = {"run": "1", "progress": 0, "progress_msg": "", "chunk_num": 0, "token_num": 0}
         DocumentService.update_by_id(id, info)
         settings.docStoreConn.delete({"doc_id": id}, search.index_name(tenant_id), dataset_id)
         TaskService.filter_delete([Task.doc_id == id])
@@ -650,11 +740,20 @@ def parse(tenant_id, dataset_id):
         doc = doc.to_dict()
         doc["tenant_id"] = tenant_id
         bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
-        queue_tasks(doc, bucket, name)
+        queue_tasks(doc, bucket, name, 0)
+        success_count += 1
+    if not_found:
+        return get_result(message=f"Documents not found: {not_found}", code=settings.RetCode.DATA_ERROR)
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(message=f"Partially parsed {success_count} documents with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages},)
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+
     return get_result()
 
 
-@manager.route("/datasets/<dataset_id>/chunks", methods=["DELETE"])
+@manager.route("/datasets/<dataset_id>/chunks", methods=["DELETE"])  # noqa: F821
 @token_required
 def stop_parsing(tenant_id, dataset_id):
     """
@@ -696,23 +795,35 @@ def stop_parsing(tenant_id, dataset_id):
     if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
     req = request.json
+
     if not req.get("document_ids"):
         return get_error_data_result("`document_ids` is required")
-    for id in req["document_ids"]:
+    doc_list = req.get("document_ids")
+    unique_doc_ids, duplicate_messages = check_duplicate_ids(doc_list, "document")
+    doc_list = unique_doc_ids
+
+    success_count = 0
+    for id in doc_list:
         doc = DocumentService.query(id=id, kb_id=dataset_id)
         if not doc:
             return get_error_data_result(message=f"You don't own the document {id}.")
-        if int(doc[0].progress) == 1 or int(doc[0].progress) == 0:
+        if int(doc[0].progress) == 1 or doc[0].progress == 0:
             return get_error_data_result(
                 "Can't stop parsing document with progress at 0 or 1"
             )
         info = {"run": "2", "progress": 0, "chunk_num": 0}
         DocumentService.update_by_id(id, info)
-        settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), dataset_id)
+        settings.docStoreConn.delete({"doc_id": doc[0].id}, search.index_name(tenant_id), dataset_id)
+        success_count += 1
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(message=f"Partially stopped {success_count} documents with {len(duplicate_messages)} errors", data={"success_count": success_count, "errors": duplicate_messages},)
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
     return get_result()
 
 
-@manager.route("/datasets/<dataset_id>/documents/<document_id>/chunks", methods=["GET"])
+@manager.route("/datasets/<dataset_id>/documents/<document_id>/chunks", methods=["GET"])  # noqa: F821
 @token_required
 def list_chunks(tenant_id, dataset_id, document_id):
     """
@@ -745,6 +856,12 @@ def list_chunks(tenant_id, dataset_id, document_id):
         required: false
         default: 30
         description: Number of items per page.
+      - in: query
+        name: id
+        type: string
+        required: false
+        default: ""
+        description: Chunk Id.
       - in: header
         name: Authorization
         type: string
@@ -827,74 +944,61 @@ def list_chunks(tenant_id, dataset_id, document_id):
             renamed_doc["run"] = run_mapping.get(str(value))
 
     res = {"total": 0, "chunks": [], "doc": renamed_doc}
-    origin_chunks = []
-    if settings.docStoreConn.indexExist(search.index_name(tenant_id), dataset_id):
+    if req.get("id"):
+        chunk = settings.docStoreConn.get(req.get("id"), search.index_name(tenant_id), [dataset_id])
+        if not chunk:
+            return get_result(message=f"Chunk not found: {dataset_id}/{req.get('id')}", code=settings.RetCode.NOT_FOUND)
+        k = []
+        for n in chunk.keys():
+            if re.search(r"(_vec$|_sm_|_tks|_ltks)", n):
+                k.append(n)
+        for n in k:
+            del chunk[n]
+        if not chunk:
+            return get_error_data_result(f"Chunk `{req.get('id')}` not found.")
+        res['total'] = 1
+        final_chunk = {
+            "id":chunk.get("id",chunk.get("chunk_id")),
+            "content":chunk["content_with_weight"],
+            "document_id":chunk.get("doc_id",chunk.get("document_id")),
+            "docnm_kwd":chunk["docnm_kwd"],
+            "important_keywords":chunk.get("important_kwd",[]),
+            "questions":chunk.get("question_kwd",[]),
+            "dataset_id":chunk.get("kb_id",chunk.get("dataset_id")),
+            "image_id":chunk.get("img_id", ""),
+            "available":bool(chunk.get("available_int",1)),
+            "positions":chunk.get("position_int",[]),
+        }
+        res["chunks"].append(final_chunk)
+        _ = Chunk(**final_chunk)
+
+    elif settings.docStoreConn.indexExist(search.index_name(tenant_id), dataset_id):
         sres = settings.retrievaler.search(query, search.index_name(tenant_id), [dataset_id], emb_mdl=None,
                                            highlight=True)
         res["total"] = sres.total
-        sign = 0
         for id in sres.ids:
             d = {
                 "id": id,
-                "content_with_weight": (
+                "content": (
                     rmSpace(sres.highlight[id])
                     if question and id in sres.highlight
                     else sres.field[id].get("content_with_weight", "")
                 ),
-                "doc_id": sres.field[id]["doc_id"],
+                "document_id": sres.field[id]["doc_id"],
                 "docnm_kwd": sres.field[id]["docnm_kwd"],
-                "important_kwd": sres.field[id].get("important_kwd", []),
-                "img_id": sres.field[id].get("img_id", ""),
-                "available_int": sres.field[id].get("available_int", 1),
-                "positions": sres.field[id].get("position_int", "").split("\t"),
+                "important_keywords": sres.field[id].get("important_kwd", []),
+                "questions": sres.field[id].get("question_kwd", []),
+                "dataset_id": sres.field[id].get("kb_id", sres.field[id].get("dataset_id")),
+                "image_id": sres.field[id].get("img_id", ""),
+                "available": bool(int(sres.field[id].get("available_int", "1"))),
+                "positions": sres.field[id].get("position_int",[]),
             }
-            if len(d["positions"]) % 5 == 0:
-                poss = []
-                for i in range(0, len(d["positions"]), 5):
-                    poss.append(
-                        [
-                            float(d["positions"][i]),
-                            float(d["positions"][i + 1]),
-                            float(d["positions"][i + 2]),
-                            float(d["positions"][i + 3]),
-                            float(d["positions"][i + 4]),
-                        ]
-                    )
-                d["positions"] = poss
-
-            origin_chunks.append(d)
-            if req.get("id"):
-                if req.get("id") == id:
-                    origin_chunks.clear()
-                    origin_chunks.append(d)
-                    sign = 1
-                    break
-        if req.get("id"):
-            if sign == 0:
-                return get_error_data_result(f"Can't find this chunk {req.get('id')}")
-
-    for chunk in origin_chunks:
-        key_mapping = {
-            "id": "id",
-            "content_with_weight": "content",
-            "doc_id": "document_id",
-            "important_kwd": "important_keywords",
-            "img_id": "image_id",
-            "available_int": "available",
-        }
-        renamed_chunk = {}
-        for key, value in chunk.items():
-            new_key = key_mapping.get(key, key)
-            renamed_chunk[new_key] = value
-        if renamed_chunk["available"] == 0:
-            renamed_chunk["available"] = False
-        if renamed_chunk["available"] == 1:
-            renamed_chunk["available"] = True
-        res["chunks"].append(renamed_chunk)
+            res["chunks"].append(d)
+            _ = Chunk(**d) # validate the chunk
     return get_result(data=res)
 
 
-@manager.route(
+@manager.route(  # noqa: F821
     "/datasets/<dataset_id>/documents/<document_id>/chunks", methods=["POST"]
 )
 @token_required
@@ -971,17 +1075,19 @@ def add_chunk(tenant_id, dataset_id, document_id):
         )
     doc = doc[0]
     req = request.json
-    if not req.get("content"):
+    if not str(req.get("content", "")).strip():
         return get_error_data_result(message="`content` is required")
     if "important_keywords" in req:
-        if type(req["important_keywords"]) != list:
+        if not isinstance(req["important_keywords"], list):
             return get_error_data_result(
                 "`important_keywords` is required to be a list"
             )
-    md5 = hashlib.md5()
-    md5.update((req["content"] + document_id).encode("utf-8"))
-
-    chunk_id = md5.hexdigest()
+    if "questions" in req:
+        if not isinstance(req["questions"], list):
+            return get_error_data_result(
+                "`questions` is required to be a list"
+            )
+    chunk_id = xxhash.xxh64((req["content"] + document_id).encode("utf-8")).hexdigest()
     d = {
         "id": chunk_id,
         "content_ltks": rag_tokenizer.tokenize(req["content"]),
@@ -992,6 +1098,10 @@ def add_chunk(tenant_id, dataset_id, document_id):
     d["important_tks"] = rag_tokenizer.tokenize(
         " ".join(req.get("important_keywords", []))
     )
+    d["question_kwd"] = [str(q).strip() for q in req.get("questions", []) if str(q).strip()]
+    d["question_tks"] = rag_tokenizer.tokenize(
+        "\n".join(req.get("questions", []))
+    )
     d["create_time"] = str(datetime.datetime.now()).replace("T", " ")[:19]
     d["create_timestamp_flt"] = datetime.datetime.now().timestamp()
     d["kb_id"] = dataset_id
@@ -1001,7 +1111,7 @@ def add_chunk(tenant_id, dataset_id, document_id):
     embd_mdl = TenantLLMService.model_instance(
         tenant_id, LLMType.EMBEDDING.value, embd_id
     )
-    v, c = embd_mdl.encode([doc.name, req["content"]])
+    v, c = embd_mdl.encode([doc.name, req["content"] if not d["question_kwd"] else "\n".join(d["question_kwd"])])
     v = 0.1 * v[0] + 0.9 * v[1]
     d["q_%d_vec" % len(v)] = v.tolist()
     settings.docStoreConn.insert([d], search.index_name(tenant_id), dataset_id)
@@ -1013,6 +1123,7 @@ def add_chunk(tenant_id, dataset_id, document_id):
         "content_with_weight": "content",
         "doc_id": "document_id",
         "important_kwd": "important_keywords",
+        "question_kwd": "questions",
         "kb_id": "dataset_id",
         "create_timestamp_flt": "create_timestamp",
         "create_time": "create_time",
@@ -1023,11 +1134,12 @@ def add_chunk(tenant_id, dataset_id, document_id):
         if key in key_mapping:
             new_key = key_mapping.get(key, key)
             renamed_chunk[new_key] = value
+    _ = Chunk(**renamed_chunk)  # validate the chunk
     return get_result(data={"chunk": renamed_chunk})
     # return get_result(data={"chunk_id": chunk_id})
 
 
-@manager.route(
+@manager.route(  # noqa: F821
     "datasets/<dataset_id>/documents/<document_id>/chunks", methods=["DELETE"]
 )
 @token_required
@@ -1075,19 +1187,27 @@ def rm_chunk(tenant_id, dataset_id, document_id):
     """
     if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
         return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    docs = DocumentService.get_by_ids([document_id])
+    if not docs:
+        raise LookupError(f"Can't find the document with ID {document_id}!")
     req = request.json
     condition = {"doc_id": document_id}
     if "chunk_ids" in req:
-        condition["id"] = req["chunk_ids"]
+        unique_chunk_ids, duplicate_messages = check_duplicate_ids(req["chunk_ids"], "chunk")
+        condition["id"] = unique_chunk_ids
     chunk_number = settings.docStoreConn.delete(condition, search.index_name(tenant_id), dataset_id)
     if chunk_number != 0:
         DocumentService.decrement_chunk_num(document_id, dataset_id, 1, chunk_number, 0)
-    if "chunk_ids" in req and chunk_number != len(req["chunk_ids"]):
-        return get_error_data_result(message=f"rm_chunk deleted chunks {chunk_number}, expect {len(req['chunk_ids'])}")
+    if "chunk_ids" in req and chunk_number != len(unique_chunk_ids):
+        if len(unique_chunk_ids) == 0:
+            return get_result(message=f"deleted {chunk_number} chunks")
+        return get_error_data_result(message=f"rm_chunk deleted chunks {chunk_number}, expect {len(unique_chunk_ids)}")
+    if duplicate_messages:
+        return get_result(message=f"Partially deleted {chunk_number} chunks with {len(duplicate_messages)} errors", data={"success_count": chunk_number, "errors": duplicate_messages},)
     return get_result(message=f"deleted {chunk_number} chunks")
 
 
-@manager.route(
+@manager.route(  # noqa: F821
     "/datasets/<dataset_id>/documents/<document_id>/chunks/<chunk_id>", methods=["PUT"]
 )
 @token_required
@@ -1166,8 +1286,13 @@ def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
     if "important_keywords" in req:
         if not isinstance(req["important_keywords"], list):
             return get_error_data_result("`important_keywords` should be a list")
-        d["important_kwd"] = req.get("important_keywords")
+        d["important_kwd"] = req.get("important_keywords", [])
         d["important_tks"] = rag_tokenizer.tokenize(" ".join(req["important_keywords"]))
+    if "questions" in req:
+        if not isinstance(req["questions"], list):
+            return get_error_data_result("`questions` should be a list")
+        d["question_kwd"] = [str(q).strip() for q in req.get("questions", []) if str(q).strip()]
+        d["question_tks"] = rag_tokenizer.tokenize("\n".join(req["questions"]))
     if "available" in req:
         d["available_int"] = int(req["available"])
     embd_id = DocumentService.get_embd_id(document_id)
@@ -1185,14 +1310,14 @@ def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
             d, arr[0], arr[1], not any([rag_tokenizer.is_chinese(t) for t in q + a])
         )
 
-    v, c = embd_mdl.encode([doc.name, d["content_with_weight"]])
+    v, c = embd_mdl.encode([doc.name, d["content_with_weight"] if not d.get("question_kwd") else "\n".join(d["question_kwd"])])
     v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
     d["q_%d_vec" % len(v)] = v.tolist()
     settings.docStoreConn.update({"id": chunk_id}, d, search.index_name(tenant_id), dataset_id)
     return get_result()
 
 
-@manager.route("/retrieval", methods=["POST"])
+@manager.route("/retrieval", methods=["POST"])  # noqa: F821
 @token_required
 def retrieval_test(tenant_id):
     """
@@ -1278,15 +1403,15 @@ def retrieval_test(tenant_id):
     kb_ids = req["dataset_ids"]
     if not isinstance(kb_ids, list):
         return get_error_data_result("`dataset_ids` should be a list")
-    kbs = KnowledgebaseService.get_by_ids(kb_ids)
     for id in kb_ids:
         if not KnowledgebaseService.accessible(kb_id=id, user_id=tenant_id):
             return get_error_data_result(f"You don't own the dataset {id}.")
-    embd_nms = list(set([kb.embd_id for kb in kbs]))
+    kbs = KnowledgebaseService.get_by_ids(kb_ids)
+    embd_nms = list(set([TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]))  # remove vendor suffix for comparison
     if len(embd_nms) != 1:
         return get_result(
             message='Datasets use different embedding models."',
-            code=settings.RetCode.AUTHENTICATION_ERROR,
+            code=settings.RetCode.DATA_ERROR,
         )
     if "question" not in req:
         return get_error_data_result("`question` is required.")
@@ -1294,6 +1419,7 @@ def retrieval_test(tenant_id):
     size = int(req.get("page_size", 30))
     question = req["question"]
     doc_ids = req.get("document_ids", [])
+    use_kg = req.get("use_kg", False)
     if not isinstance(doc_ids, list):
         return get_error_data_result("`documents` should be a list")
     doc_ids_list = KnowledgebaseService.list_documents_by_ids(kb_ids)
@@ -1310,28 +1436,24 @@ def retrieval_test(tenant_id):
     else:
         highlight = True
     try:
+        tenant_ids = list(set([kb.tenant_id for kb in kbs]))
         e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
         if not e:
             return get_error_data_result(message="Dataset not found!")
-        embd_mdl = TenantLLMService.model_instance(
-            kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id
-        )
+        embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING, llm_name=kb.embd_id)
 
         rerank_mdl = None
         if req.get("rerank_id"):
-            rerank_mdl = TenantLLMService.model_instance(
-                kb.tenant_id, LLMType.RERANK.value, llm_name=req["rerank_id"]
-            )
+            rerank_mdl = LLMBundle(kb.tenant_id, LLMType.RERANK, llm_name=req["rerank_id"])
 
         if req.get("keyword", False):
-            chat_mdl = TenantLLMService.model_instance(kb.tenant_id, LLMType.CHAT)
+            chat_mdl = LLMBundle(kb.tenant_id, LLMType.CHAT)
             question += keyword_extraction(chat_mdl, question)
 
-        retr = settings.retrievaler if kb.parser_id != ParserType.KG else settings.kg_retrievaler
-        ranks = retr.retrieval(
+        ranks = settings.retrievaler.retrieval(
             question,
             embd_mdl,
-            kb.tenant_id,
+            tenant_ids,
             kb_ids,
             page,
             size,
@@ -1341,7 +1463,17 @@ def retrieval_test(tenant_id):
             doc_ids,
             rerank_mdl=rerank_mdl,
             highlight=highlight,
+            rank_feature=label_question(question, kbs)
         )
+        if use_kg:
+            ck = settings.kg_retrievaler.retrieval(question,
+                                                   [k.tenant_id for k in kbs],
+                                                   kb_ids,
+                                                   embd_mdl,
+                                                   LLMBundle(kb.tenant_id, LLMType.CHAT))
+            if ck["content_with_weight"]:
+                ranks["chunks"].insert(0, ck)
+
         for c in ranks["chunks"]:
             c.pop("vector", None)
 
@@ -1353,7 +1485,9 @@ def retrieval_test(tenant_id):
                 "content_with_weight": "content",
                 "doc_id": "document_id",
                 "important_kwd": "important_keywords",
+                "question_kwd": "questions",
                 "docnm_kwd": "document_keyword",
+                "kb_id":"dataset_id"
             }
             rename_chunk = {}
             for key, value in chunk.items():

api/apps/system_app.py

@@ -37,8 +37,7 @@ from timeit import default_timer as timer
 
 from rag.utils.redis_conn import REDIS_CONN
 
-
-@manager.route("/version", methods=["GET"])
+@manager.route("/version", methods=["GET"])  # noqa: F821
 @login_required
 def version():
     """
@@ -61,7 +60,7 @@ def version():
     return get_json_result(data=get_ragflow_version())
 
 
-@manager.route("/status", methods=["GET"])
+@manager.route("/status", methods=["GET"])  # noqa: F821
 @login_required
 def status():
     """
@@ -170,7 +169,7 @@ def status():
     return get_json_result(data=res)
 
 
-@manager.route("/new_token", methods=["POST"])
+@manager.route("/new_token", methods=["POST"])  # noqa: F821
 @login_required
 def new_token():
     """
@@ -201,10 +200,11 @@ def new_token():
         if not tenants:
             return get_data_error_result(message="Tenant not found!")
 
-        tenant_id = tenants[0].tenant_id
+        tenant_id = [tenant for tenant in tenants if tenant.role == 'owner'][0].tenant_id
         obj = {
             "tenant_id": tenant_id,
             "token": generate_confirmation_token(tenant_id),
+            "beta": generate_confirmation_token(generate_confirmation_token(tenant_id)).replace("ragflow-", "")[:32],
             "create_time": current_timestamp(),
             "create_date": datetime_format(datetime.now()),
             "update_time": None,
@@ -219,7 +219,7 @@ def new_token():
         return server_error_response(e)
 
 
-@manager.route("/token_list", methods=["GET"])
+@manager.route("/token_list", methods=["GET"])  # noqa: F821
 @login_required
 def token_list():
     """
@@ -255,13 +255,19 @@ def token_list():
         if not tenants:
             return get_data_error_result(message="Tenant not found!")
 
-        objs = APITokenService.query(tenant_id=tenants[0].tenant_id)
-        return get_json_result(data=[o.to_dict() for o in objs])
+        tenant_id = [tenant for tenant in tenants if tenant.role == 'owner'][0].tenant_id
+        objs = APITokenService.query(tenant_id=tenant_id)
+        objs = [o.to_dict() for o in objs]
+        for o in objs:
+            if not o["beta"]:
+                o["beta"] = generate_confirmation_token(generate_confirmation_token(tenants[0].tenant_id)).replace("ragflow-", "")[:32]
+                APITokenService.filter_update([APIToken.tenant_id == tenant_id, APIToken.token == o["token"]], o)
+        return get_json_result(data=objs)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route("/token/<token>", methods=["DELETE"])
+@manager.route("/token/<token>", methods=["DELETE"])  # noqa: F821
 @login_required
 def rm(token):
     """
@@ -291,3 +297,25 @@ def rm(token):
         [APIToken.tenant_id == current_user.id, APIToken.token == token]
     )
     return get_json_result(data=True)
+
+
+@manager.route('/config', methods=['GET'])  # noqa: F821
+def get_config():
+    """
+    Get system configuration.
+    ---
+    tags:
+        - System
+    responses:
+        200:
+            description: Return system configuration
+            schema:
+                type: object
+                properties:
+                    registerEnable:
+                        type: integer 0 means disabled, 1 means enabled
+                        description: Whether user registration is enabled
+    """
+    return get_json_result(data={
+        "registerEnabled": settings.REGISTER_ENABLED
+    })

api/apps/tenant_app.py

@@ -26,7 +26,7 @@ from api.utils import get_uuid, delta_seconds
 from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result
 
 
-@manager.route("/<tenant_id>/user/list", methods=["GET"])
+@manager.route("/<tenant_id>/user/list", methods=["GET"])  # noqa: F821
 @login_required
 def user_list(tenant_id):
     if current_user.id != tenant_id:
@@ -44,7 +44,7 @@ def user_list(tenant_id):
         return server_error_response(e)
 
 
-@manager.route('/<tenant_id>/user', methods=['POST'])
+@manager.route('/<tenant_id>/user', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("email")
 def create(tenant_id):
@@ -55,32 +55,36 @@ def create(tenant_id):
             code=settings.RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
-    usrs = UserService.query(email=req["email"])
-    if not usrs:
+    invite_user_email = req["email"]
+    invite_users = UserService.query(email=invite_user_email)
+    if not invite_users:
         return get_data_error_result(message="User not found.")
 
-    user_id = usrs[0].id
-    user_tenants = UserTenantService.query(user_id=user_id, tenant_id=tenant_id)
+    user_id_to_invite = invite_users[0].id
+    user_tenants = UserTenantService.query(user_id=user_id_to_invite, tenant_id=tenant_id)
     if user_tenants:
-        if user_tenants[0].status == UserTenantRole.NORMAL.value:
-            return get_data_error_result(message="This user is in the team already.")
-        return get_data_error_result(message="Invitation notification is sent.")
+        user_tenant_role = user_tenants[0].role
+        if user_tenant_role == UserTenantRole.NORMAL:
+            return get_data_error_result(message=f"{invite_user_email} is already in the team.")
+        if user_tenant_role == UserTenantRole.OWNER:
+            return get_data_error_result(message=f"{invite_user_email} is the owner of the team.")
+        return get_data_error_result(message=f"{invite_user_email} is in the team, but the role: {user_tenant_role} is invalid.")
 
     UserTenantService.save(
         id=get_uuid(),
-        user_id=user_id,
+        user_id=user_id_to_invite,
         tenant_id=tenant_id,
         invited_by=current_user.id,
         role=UserTenantRole.INVITE,
         status=StatusEnum.VALID.value)
 
-    usr = usrs[0].to_dict()
+    usr = invite_users[0].to_dict()
     usr = {k: v for k, v in usr.items() if k in ["id", "avatar", "email", "nickname"]}
 
     return get_json_result(data=usr)
 
 
-@manager.route('/<tenant_id>/user/<user_id>', methods=['DELETE'])
+@manager.route('/<tenant_id>/user/<user_id>', methods=['DELETE'])  # noqa: F821
 @login_required
 def rm(tenant_id, user_id):
     if current_user.id != tenant_id and current_user.id != user_id:
@@ -96,7 +100,7 @@ def rm(tenant_id, user_id):
         return server_error_response(e)
 
 
-@manager.route("/list", methods=["GET"])
+@manager.route("/list", methods=["GET"])  # noqa: F821
 @login_required
 def tenant_list():
     try:
@@ -108,7 +112,7 @@ def tenant_list():
         return server_error_response(e)
 
 
-@manager.route("/agree/<tenant_id>", methods=["PUT"])
+@manager.route("/agree/<tenant_id>", methods=["PUT"])  # noqa: F821
 @login_required
 def agree(tenant_id):
     try:

api/apps/user_app.py

@@ -13,38 +13,40 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import logging
 import json
+import logging
 import re
 from datetime import datetime
 
-from flask import request, session, redirect
-from werkzeug.security import generate_password_hash, check_password_hash
-from flask_login import login_required, current_user, login_user, logout_user
+from flask import redirect, request, session
+from flask_login import current_user, login_required, login_user, logout_user
+from werkzeug.security import check_password_hash, generate_password_hash
 
+from api import settings
+from api.apps.auth import get_auth_client
+from api.db import FileType, UserTenantRole
 from api.db.db_models import TenantLLM
-from api.db.services.llm_service import TenantLLMService, LLMService
-from api.utils.api_utils import (
-    server_error_response,
-    validate_request,
-    get_data_error_result,
-)
+from api.db.services.file_service import FileService
+from api.db.services.llm_service import LLMService, TenantLLMService
+from api.db.services.user_service import TenantService, UserService, UserTenantService
 from api.utils import (
-    get_uuid,
-    get_format_time,
-    decrypt,
-    download_img,
     current_timestamp,
     datetime_format,
+    decrypt,
+    download_img,
+    get_format_time,
+    get_uuid,
+)
+from api.utils.api_utils import (
+    construct_response,
+    get_data_error_result,
+    get_json_result,
+    server_error_response,
+    validate_request,
 )
-from api.db import UserTenantRole, FileType
-from api import settings
-from api.db.services.user_service import UserService, TenantService, UserTenantService
-from api.db.services.file_service import FileService
-from api.utils.api_utils import get_json_result, construct_response
 
 
-@manager.route("/login", methods=["POST", "GET"])
+@manager.route("/login", methods=["POST", "GET"])  # noqa: F821
 def login():
     """
     User login endpoint.
@@ -76,9 +78,7 @@ def login():
           type: object
     """
     if not request.json:
-        return get_json_result(
-            data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="Unauthorized!"
-        )
+        return get_json_result(data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="Unauthorized!")
 
     email = request.json.get("email", "")
     users = UserService.query(email=email)
@@ -93,9 +93,7 @@ def login():
     try:
         password = decrypt(password)
     except BaseException:
-        return get_json_result(
-            data=False, code=settings.RetCode.SERVER_ERROR, message="Fail to crypt password"
-        )
+        return get_json_result(data=False, code=settings.RetCode.SERVER_ERROR, message="Fail to crypt password")
 
     user = UserService.query_user(email, password)
     if user:
@@ -115,9 +113,131 @@ def login():
         )
 
 
-@manager.route("/github_callback", methods=["GET"])
+@manager.route("/login/channels", methods=["GET"])  # noqa: F821
+def get_login_channels():
+    """
+    Get all supported authentication channels.
+    """
+    try:
+        channels = []
+        for channel, config in settings.OAUTH_CONFIG.items():
+            channels.append(
+                {
+                    "channel": channel,
+                    "display_name": config.get("display_name", channel.title()),
+                    "icon": config.get("icon", "sso"),
+                }
+            )
+        return get_json_result(data=channels)
+    except Exception as e:
+        logging.exception(e)
+        return get_json_result(data=[], message=f"Load channels failure, error: {str(e)}", code=settings.RetCode.EXCEPTION_ERROR)
+
+
+@manager.route("/login/<channel>", methods=["GET"])  # noqa: F821
+def oauth_login(channel):
+    channel_config = settings.OAUTH_CONFIG.get(channel)
+    if not channel_config:
+        raise ValueError(f"Invalid channel name: {channel}")
+    auth_cli = get_auth_client(channel_config)
+
+    state = get_uuid()
+    session["oauth_state"] = state
+    auth_url = auth_cli.get_authorization_url(state)
+    return redirect(auth_url)
+
+
+@manager.route("/oauth/callback/<channel>", methods=["GET"])  # noqa: F821
+def oauth_callback(channel):
+    """
+    Handle the OAuth/OIDC callback for various channels dynamically.
+    """
+    try:
+        channel_config = settings.OAUTH_CONFIG.get(channel)
+        if not channel_config:
+            raise ValueError(f"Invalid channel name: {channel}")
+        auth_cli = get_auth_client(channel_config)
+
+        # Check the state
+        state = request.args.get("state")
+        if not state or state != session.get("oauth_state"):
+            return redirect("/?error=invalid_state")
+        session.pop("oauth_state", None)
+
+        # Obtain the authorization code
+        code = request.args.get("code")
+        if not code:
+            return redirect("/?error=missing_code")
+
+        # Exchange authorization code for access token
+        token_info = auth_cli.exchange_code_for_token(code)
+        access_token = token_info.get("access_token")
+        if not access_token:
+            return redirect("/?error=token_failed")
+
+        id_token = token_info.get("id_token")
+
+        # Fetch user info
+        user_info = auth_cli.fetch_user_info(access_token, id_token=id_token)
+        if not user_info.email:
+            return redirect("/?error=email_missing")
+
+        # Login or register
+        users = UserService.query(email=user_info.email)
+        user_id = get_uuid()
+
+        if not users:
+            try:
+                try:
+                    avatar = download_img(user_info.avatar_url)
+                except Exception as e:
+                    logging.exception(e)
+                    avatar = ""
+
+                users = user_register(
+                    user_id,
+                    {
+                        "access_token": get_uuid(),
+                        "email": user_info.email,
+                        "avatar": avatar,
+                        "nickname": user_info.nickname,
+                        "login_channel": channel,
+                        "last_login_time": get_format_time(),
+                        "is_superuser": False,
+                    },
+                )
+
+                if not users:
+                    raise Exception(f"Failed to register {user_info.email}")
+                if len(users) > 1:
+                    raise Exception(f"Same email: {user_info.email} exists!")
+
+                # Try to log in
+                user = users[0]
+                login_user(user)
+                return redirect(f"/?auth={user.get_id()}")
+
+            except Exception as e:
+                rollback_user_registration(user_id)
+                logging.exception(e)
+                return redirect(f"/?error={str(e)}")
+
+        # User exists, try to log in
+        user = users[0]
+        user.access_token = get_uuid()
+        login_user(user)
+        user.save()
+        return redirect(f"/?auth={user.get_id()}")
+    except Exception as e:
+        logging.exception(e)
+        return redirect(f"/?error={str(e)}")
+
+
+@manager.route("/github_callback", methods=["GET"])  # noqa: F821
 def github_callback():
     """
+    **Deprecated**, Use `/oauth/callback/<channel>` instead.
+
     GitHub OAuth callback endpoint.
     ---
     tags:
@@ -200,7 +320,7 @@ def github_callback():
     return redirect("/?auth=%s" % user.get_id())
 
 
-@manager.route("/feishu_callback", methods=["GET"])
+@manager.route("/feishu_callback", methods=["GET"])  # noqa: F821
 def feishu_callback():
     """
     Feishu OAuth callback endpoint.
@@ -252,7 +372,7 @@ def feishu_callback():
     if res["code"] != 0:
         return redirect("/?error=%s" % res["message"])
 
-    if "contact:user.email:readonly" not in res["data"]["scope"].split(" "):
+    if "contact:user.email:readonly" not in res["data"]["scope"].split():
         return redirect("/?error=contact:user.email:readonly not in scope")
     session["access_token"] = res["data"]["access_token"]
     session["access_token_from"] = "feishu"
@@ -309,9 +429,7 @@ def user_info_from_feishu(access_token):
         "Content-Type": "application/json; charset=utf-8",
         "Authorization": f"Bearer {access_token}",
     }
-    res = requests.get(
-        "https://open.feishu.cn/open-apis/authen/v1/user_info", headers=headers
-    )
+    res = requests.get("https://open.feishu.cn/open-apis/authen/v1/user_info", headers=headers)
     user_info = res.json()["data"]
     user_info["email"] = None if user_info.get("email") == "" else user_info["email"]
     return user_info
@@ -321,21 +439,17 @@ def user_info_from_github(access_token):
     import requests
 
     headers = {"Accept": "application/json", "Authorization": f"token {access_token}"}
-    res = requests.get(
-        f"https://api.github.com/user?access_token={access_token}", headers=headers
-    )
+    res = requests.get(f"https://api.github.com/user?access_token={access_token}", headers=headers)
     user_info = res.json()
     email_info = requests.get(
         f"https://api.github.com/user/emails?access_token={access_token}",
         headers=headers,
     ).json()
-    user_info["email"] = next(
-        (email for email in email_info if email["primary"] == True), None
-    )["email"]
+    user_info["email"] = next((email for email in email_info if email["primary"]), None)["email"]
     return user_info
 
 
-@manager.route("/logout", methods=["GET"])
+@manager.route("/logout", methods=["GET"])  # noqa: F821
 @login_required
 def log_out():
     """
@@ -357,7 +471,7 @@ def log_out():
     return get_json_result(data=True)
 
 
-@manager.route("/setting", methods=["POST"])
+@manager.route("/setting", methods=["POST"])  # noqa: F821
 @login_required
 def setting_user():
     """
@@ -391,9 +505,7 @@ def setting_user():
     request_data = request.json
     if request_data.get("password"):
         new_password = request_data.get("new_password")
-        if not check_password_hash(
-                current_user.password, decrypt(request_data["password"])
-        ):
+        if not check_password_hash(current_user.password, decrypt(request_data["password"])):
             return get_json_result(
                 data=False,
                 code=settings.RetCode.AUTHENTICATION_ERROR,
@@ -424,12 +536,10 @@ def setting_user():
         return get_json_result(data=True)
     except Exception as e:
         logging.exception(e)
-        return get_json_result(
-            data=False, message="Update failure!", code=settings.RetCode.EXCEPTION_ERROR
-        )
+        return get_json_result(data=False, message="Update failure!", code=settings.RetCode.EXCEPTION_ERROR)
 
 
-@manager.route("/info", methods=["GET"])
+@manager.route("/info", methods=["GET"])  # noqa: F821
 @login_required
 def user_profile():
     """
@@ -518,9 +628,23 @@ def user_register(user_id, user):
                 "model_type": llm.model_type,
                 "api_key": settings.API_KEY,
                 "api_base": settings.LLM_BASE_URL,
-                "max_tokens": llm.max_tokens if llm.max_tokens else 8192
+                "max_tokens": llm.max_tokens if llm.max_tokens else 8192,
             }
         )
+    if settings.LIGHTEN != 1:
+        for buildin_embedding_model in settings.BUILTIN_EMBEDDING_MODELS:
+            mdlnm, fid = TenantLLMService.split_model_name_and_factory(buildin_embedding_model)
+            tenant_llm.append(
+                {
+                    "tenant_id": user_id,
+                    "llm_factory": fid,
+                    "llm_name": mdlnm,
+                    "model_type": "embedding",
+                    "api_key": "",
+                    "api_base": "",
+                    "max_tokens": 1024 if buildin_embedding_model == "BAAI/bge-large-zh-v1.5@BAAI" else 512,
+                }
+            )
 
     if not UserService.save(**user):
         return
@@ -531,7 +655,7 @@ def user_register(user_id, user):
     return UserService.query(email=user["email"])
 
 
-@manager.route("/register", methods=["POST"])
+@manager.route("/register", methods=["POST"])  # noqa: F821
 @validate_request("nickname", "email", "password")
 def user_add():
     """
@@ -562,11 +686,19 @@ def user_add():
         schema:
           type: object
     """
+
+    if not settings.REGISTER_ENABLED:
+        return get_json_result(
+            data=False,
+            message="User registration is disabled!",
+            code=settings.RetCode.OPERATING_ERROR,
+        )
+
     req = request.json
     email_address = req["email"]
 
     # Validate the email address
-    if not re.match(r"^[\w\._-]+@([\w_-]+\.)+[\w-]{2,5}$", email_address):
+    if not re.match(r"^[\w\._-]+@([\w_-]+\.)+[\w-]{2,}$", email_address):
         return get_json_result(
             data=False,
             message=f"Invalid email address: {email_address}!",
@@ -617,7 +749,7 @@ def user_add():
         )
 
 
-@manager.route("/tenant_info", methods=["GET"])
+@manager.route("/tenant_info", methods=["GET"])  # noqa: F821
 @login_required
 def tenant_info():
     """
@@ -655,7 +787,7 @@ def tenant_info():
         return server_error_response(e)
 
 
-@manager.route("/set_tenant_info", methods=["POST"])
+@manager.route("/set_tenant_info", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("tenant_id", "asr_id", "embd_id", "img2txt_id", "llm_id")
 def set_tenant_info():

api/constants.py

@@ -23,3 +23,5 @@ API_VERSION = "v1"
 RAG_FLOW_SERVICE_NAME = "ragflow"
 REQUEST_WAIT_SEC = 2
 REQUEST_MAX_WAIT_SEC = 300
+
+DATASET_NAME_LIMIT = 128

api/db/__init__.py

@@ -49,6 +49,7 @@ class FileType(StrEnum):
     FOLDER = 'folder'
     OTHER = "other"
 
+VALID_FILE_TYPES = {FileType.PDF, FileType.DOC, FileType.VISUAL, FileType.AURAL, FileType.VIRTUAL, FileType.FOLDER, FileType.OTHER}
 
 class LLMType(StrEnum):
     CHAT = 'chat'
@@ -73,6 +74,7 @@ class TaskStatus(StrEnum):
     DONE = "3"
     FAIL = "4"
 
+VALID_TASK_STATUS     = {TaskStatus.UNSTART, TaskStatus.RUNNING, TaskStatus.CANCEL, TaskStatus.DONE, TaskStatus.FAIL}
 
 class ParserType(StrEnum):
     PRESENTATION = "presentation"
@@ -89,6 +91,7 @@ class ParserType(StrEnum):
     AUDIO = "audio"
     EMAIL = "email"
     KG = "knowledge_graph"
+    TAG = "tag"
 
 
 class FileSource(StrEnum):