Docs: Update version references to v0.22.0 in READMEs and docs (#11211)

### What problem does this PR solve?

- Update version tags in README files (including translations) from
v0.21.1 to v0.22.0
- Modify Docker image references and documentation to reflect new
version
- Update version badges and image descriptions
- Maintain consistency across all language variants of README files

### Type of change

- [x] Documentation Update

.github/workflows/release.yml

@@ -16,52 +16,52 @@ concurrency:
 
 jobs:
   release:
-    runs-on: [ "self-hosted", "overseas" ]
+    runs-on: [ "self-hosted", "ragflow-test" ]
     steps:
       - name: Ensure workspace ownership
-        run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
+        run: echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
 
       # https://github.com/actions/checkout/blob/v3/README.md
       - name: Check out code
         uses: actions/checkout@v4
         with:
-          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+          token: ${{ secrets.GITHUB_TOKEN }}  # Use the secret as an environment variable
           fetch-depth: 0
           fetch-tags: true
 
       - name: Prepare release body
         run: |
-          if [[ $GITHUB_EVENT_NAME == 'create' ]]; then
+          if [[ ${GITHUB_EVENT_NAME} == "create" ]]; then
             RELEASE_TAG=${GITHUB_REF#refs/tags/}
-            if [[ $RELEASE_TAG == 'nightly' ]]; then
+            if [[ ${RELEASE_TAG} == "nightly" ]]; then
               PRERELEASE=true
             else
               PRERELEASE=false
             fi
-            echo "Workflow triggered by create tag: $RELEASE_TAG"
+            echo "Workflow triggered by create tag: ${RELEASE_TAG}"
           else
             RELEASE_TAG=nightly
             PRERELEASE=true
             echo "Workflow triggered by schedule"
           fi
-          echo "RELEASE_TAG=$RELEASE_TAG" >> $GITHUB_ENV
-          echo "PRERELEASE=$PRERELEASE" >> $GITHUB_ENV
+          echo "RELEASE_TAG=${RELEASE_TAG}" >> ${GITHUB_ENV}
+          echo "PRERELEASE=${PRERELEASE}" >> ${GITHUB_ENV}
           RELEASE_DATETIME=$(date --rfc-3339=seconds)
-          echo Release $RELEASE_TAG created from $GITHUB_SHA at $RELEASE_DATETIME > release_body.md
+          echo Release ${RELEASE_TAG} created from ${GITHUB_SHA} at ${RELEASE_DATETIME} > release_body.md
 
       - name: Move the existing mutable tag
         # https://github.com/softprops/action-gh-release/issues/171
         run: |
           git fetch --tags
-          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
+          if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
             # Determine if a given tag exists and matches a specific Git commit.
             # actions/checkout@v4 fetch-tags doesn't work when triggered by schedule
-            if [ "$(git rev-parse -q --verify "refs/tags/$RELEASE_TAG")" = "$GITHUB_SHA" ]; then
-              echo "mutable tag $RELEASE_TAG exists and matches $GITHUB_SHA"
+            if [ "$(git rev-parse -q --verify "refs/tags/${RELEASE_TAG}")" = "${GITHUB_SHA}" ]; then
+              echo "mutable tag ${RELEASE_TAG} exists and matches ${GITHUB_SHA}"
             else
-              git tag -f $RELEASE_TAG $GITHUB_SHA
-              git push -f origin $RELEASE_TAG:refs/tags/$RELEASE_TAG
-              echo "created/moved mutable tag $RELEASE_TAG to $GITHUB_SHA"
+              git tag -f ${RELEASE_TAG} ${GITHUB_SHA}
+              git push -f origin ${RELEASE_TAG}:refs/tags/${RELEASE_TAG}
+              echo "created/moved mutable tag ${RELEASE_TAG} to ${GITHUB_SHA}"
             fi
           fi
 
@@ -69,50 +69,26 @@ jobs:
         # https://github.com/actions/upload-release-asset has been replaced by https://github.com/softprops/action-gh-release
         uses: softprops/action-gh-release@v2
         with:
-          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+          token: ${{ secrets.GITHUB_TOKEN }}  # Use the secret as an environment variable
           prerelease: ${{ env.PRERELEASE }}
           tag_name: ${{ env.RELEASE_TAG }}
           # The body field does not support environment variable substitution directly.
           body_path: release_body.md
 
-      # https://github.com/marketplace/actions/docker-login
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: infiniflow
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      # https://github.com/marketplace/actions/build-and-push-docker-images
-      - name: Build and push full image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}
-          file: Dockerfile
-          platforms: linux/amd64
-
-      # https://github.com/marketplace/actions/build-and-push-docker-images
-      - name: Build and push slim image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}-slim
-          file: Dockerfile
-          build-args: LIGHTEN=1
-          platforms: linux/amd64
-
-      - name: Build ragflow-sdk
+      - name: Build and push ragflow-sdk
         if: startsWith(github.ref, 'refs/tags/v')
         run: |
-          cd sdk/python && \
-          uv build
+          cd sdk/python && uv build && uv publish --token ${{ secrets.PYPI_API_TOKEN }}
 
-      - name: Publish package distributions to PyPI
+      - name: Build and push ragflow-cli
         if: startsWith(github.ref, 'refs/tags/v')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: sdk/python/dist/
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          verbose: true
+        run: |
+          cd admin/client && uv build && uv publish --token ${{ secrets.PYPI_API_TOKEN }}
+
+      - name: Build and push image
+        run: |
+          sudo docker login --username infiniflow --password-stdin <<< ${{ secrets.DOCKERHUB_TOKEN }}
+          sudo docker build --build-arg NEED_MIRROR=1 -t infiniflow/ragflow:${RELEASE_TAG} -f Dockerfile .
+          sudo docker tag infiniflow/ragflow:${RELEASE_TAG} infiniflow/ragflow:latest
+          sudo docker push infiniflow/ragflow:${RELEASE_TAG}
+          sudo docker push infiniflow/ragflow:latest

.github/workflows/tests.yml

@@ -9,8 +9,11 @@ on:
       - 'docs/**'
       - '*.md'
       - '*.mdx'
-  pull_request:
-    types: [ opened, synchronize, reopened, labeled ]
+  # The only difference between pull_request and pull_request_target is the context in which the workflow runs:
+  # — pull_request_target workflows use the workflow files from the default branch, and secrets are available.
+  # — pull_request workflows use the workflow files from the pull request branch, and secrets are unavailable.
+  pull_request_target:
+    types: [ synchronize, ready_for_review ]
     paths-ignore:
       - 'docs/**'
       - '*.md'
@@ -28,26 +31,63 @@ jobs:
     name: ragflow_tests
     # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution
     # https://github.com/orgs/community/discussions/26261
-    if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }}
-    runs-on: [ "self-hosted", "debug" ]
+    if: ${{ github.event_name != 'pull_request_target' || contains(github.event.pull_request.labels.*.name, 'ci') }}
+    runs-on: [ "self-hosted", "ragflow-test" ]
     steps:
       # https://github.com/hmarr/debug-action
       #- uses: hmarr/debug-action@v2
 
-      - name: Show who triggered this workflow
+      - name: Ensure workspace ownership
         run: |
           echo "Workflow triggered by ${{ github.event_name }}"
-
-      - name: Ensure workspace ownership
-        run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
+          echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
 
       # https://github.com/actions/checkout/issues/1781
       - name: Check out code
         uses: actions/checkout@v4
         with:
+          ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
           fetch-depth: 0
           fetch-tags: true
 
+      - name: Check workflow duplication
+        if: ${{ !cancelled() && !failure() }}
+        run: |
+          if [[ ${GITHUB_EVENT_NAME} != "pull_request_target" && ${GITHUB_EVENT_NAME} != "schedule" ]]; then
+            HEAD=$(git rev-parse HEAD)
+            # Find a PR that introduced a given commit
+            gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}"
+            PR_NUMBER=$(gh pr list --search ${HEAD} --state merged --json number --jq .[0].number)
+            echo "HEAD=${HEAD}"
+            echo "PR_NUMBER=${PR_NUMBER}"
+            if [[ -n "${PR_NUMBER}" ]]; then
+              PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
+              if [[ -f "${PR_SHA_FP}" ]]; then
+                read -r PR_SHA PR_RUN_ID < "${PR_SHA_FP}"
+                # Calculate the hash of the current workspace content
+                HEAD_SHA=$(git rev-parse HEAD^{tree})
+                if [[ "${HEAD_SHA}" == "${PR_SHA}" ]]; then
+                  echo "Cancel myself since the workspace content hash is the same with PR #${PR_NUMBER} merged. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${PR_RUN_ID} for details."
+                  gh run cancel ${GITHUB_RUN_ID}
+                  while true; do
+                    status=$(gh run view ${GITHUB_RUN_ID} --json status -q .status)
+                    [ "${status}" = "completed" ] && break
+                    sleep 5
+                  done
+                  exit 1
+                fi
+              fi
+            fi
+          elif [[ ${GITHUB_EVENT_NAME} == "pull_request_target" ]]; then
+            PR_NUMBER=${{ github.event.pull_request.number }}
+            PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
+            # Calculate the hash of the current workspace content
+            PR_SHA=$(git rev-parse HEAD^{tree})
+            echo "PR #${PR_NUMBER} workspace content hash: ${PR_SHA}"
+            mkdir -p ${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}
+            echo "${PR_SHA} ${GITHUB_RUN_ID}" > ${PR_SHA_FP}
+          fi
+
       # https://github.com/astral-sh/ruff-action
       - name: Static check with Ruff
         uses: astral-sh/ruff-action@v3
@@ -55,122 +95,145 @@ jobs:
           version: ">=0.11.x"
           args: "check"
 
-      - name: Build ragflow:nightly-slim
-        run: |
-          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
-          sudo docker pull ubuntu:22.04
-          sudo docker build --progress=plain --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-
       - name: Build ragflow:nightly
         run: |
-          sudo docker build --progress=plain --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
-
-      - name: Start ragflow:nightly-slim
-        run: |
-          sudo docker compose -f docker/docker-compose.yml down --volumes --remove-orphans
-          echo -e "\nRAGFLOW_IMAGE=infiniflow/ragflow:nightly-slim" >> docker/.env
-          sudo docker compose -f docker/docker-compose.yml up -d
-
-      - name: Stop ragflow:nightly-slim
-        if: always()  # always run this step even if previous steps failed
-        run: |
-          sudo docker compose -f docker/docker-compose.yml down -v
+          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
+          RAGFLOW_IMAGE=infiniflow/ragflow:${GITHUB_RUN_ID}
+          echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> ${GITHUB_ENV}
+          sudo docker pull ubuntu:22.04
+          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t ${RAGFLOW_IMAGE} .
+          if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          echo "HTTP_API_TEST_LEVEL=${HTTP_API_TEST_LEVEL}" >> ${GITHUB_ENV}
+          echo "RAGFLOW_CONTAINER=${GITHUB_RUN_ID}-ragflow-cpu-1" >> ${GITHUB_ENV}
 
       - name: Start ragflow:nightly
         run: |
-          echo -e "\nRAGFLOW_IMAGE=infiniflow/ragflow:nightly" >> docker/.env
-          sudo docker compose -f docker/docker-compose.yml up -d
+          # Determine runner number (default to 1 if not found)
+          RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true)
+          RUNNER_NUM=${RUNNER_NUM:-1}
+
+          # Compute port numbers using bash arithmetic
+          ES_PORT=$((1200 + RUNNER_NUM * 10))
+          OS_PORT=$((1201 + RUNNER_NUM * 10))
+          INFINITY_THRIFT_PORT=$((23817 + RUNNER_NUM * 10))
+          INFINITY_HTTP_PORT=$((23820 + RUNNER_NUM * 10))
+          INFINITY_PSQL_PORT=$((5432 + RUNNER_NUM * 10))
+          MYSQL_PORT=$((5455 + RUNNER_NUM * 10))
+          MINIO_PORT=$((9000 + RUNNER_NUM * 10))
+          MINIO_CONSOLE_PORT=$((9001 + RUNNER_NUM * 10))
+          REDIS_PORT=$((6379 + RUNNER_NUM * 10))
+          TEI_PORT=$((6380 + RUNNER_NUM * 10))
+          KIBANA_PORT=$((6601 + RUNNER_NUM * 10))
+          SVR_HTTP_PORT=$((9380 + RUNNER_NUM * 10))
+          ADMIN_SVR_HTTP_PORT=$((9381 + RUNNER_NUM * 10))
+          SVR_MCP_PORT=$((9382 + RUNNER_NUM * 10))
+          SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + RUNNER_NUM * 10))
+          SVR_WEB_HTTP_PORT=$((80 + RUNNER_NUM * 10))
+          SVR_WEB_HTTPS_PORT=$((443 + RUNNER_NUM * 10))
+
+          # Persist computed ports into docker/.env so docker-compose uses the correct host bindings
+          echo "" >> docker/.env
+          echo -e "ES_PORT=${ES_PORT}" >> docker/.env
+          echo -e "OS_PORT=${OS_PORT}" >> docker/.env
+          echo -e "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}" >> docker/.env
+          echo -e "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}" >> docker/.env
+          echo -e "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}" >> docker/.env
+          echo -e "MYSQL_PORT=${MYSQL_PORT}" >> docker/.env
+          echo -e "MINIO_PORT=${MINIO_PORT}" >> docker/.env
+          echo -e "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}" >> docker/.env
+          echo -e "REDIS_PORT=${REDIS_PORT}" >> docker/.env
+          echo -e "TEI_PORT=${TEI_PORT}" >> docker/.env
+          echo -e "KIBANA_PORT=${KIBANA_PORT}" >> docker/.env
+          echo -e "SVR_HTTP_PORT=${SVR_HTTP_PORT}" >> docker/.env
+          echo -e "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}" >> docker/.env
+          echo -e "SVR_MCP_PORT=${SVR_MCP_PORT}" >> docker/.env
+          echo -e "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}" >> docker/.env
+          echo -e "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}" >> docker/.env
+          echo -e "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}" >> docker/.env
+
+          echo -e "COMPOSE_PROFILES=\${COMPOSE_PROFILES},tei-cpu" >> docker/.env
+          echo -e "TEI_MODEL=BAAI/bge-small-en-v1.5" >> docker/.env
+          echo -e "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> docker/.env
+          echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV}
+
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} up -d
+          uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python
 
       - name: Run sdk tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
-            export HTTP_API_TEST_LEVEL=p3
-          else
-            export HTTP_API_TEST_LEVEL=p2
-          fi
-          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python && uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
+          source .venv/bin/activate && pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
 
       - name: Run frontend api tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && UV_LINK_MODE=copy uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
+          source .venv/bin/activate && pytest -s --tb=short sdk/python/test/test_frontend_api/get_email.py sdk/python/test/test_frontend_api/test_dataset.py
           
       - name: Run http api tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
-            export HTTP_API_TEST_LEVEL=p3
-          else
-            export HTTP_API_TEST_LEVEL=p2
-          fi
-          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
+          source .venv/bin/activate && pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
         run: |
-          sudo docker compose -f docker/docker-compose.yml down -v
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v || true
+          sudo docker ps -a --filter "label=com.docker.compose.project=${GITHUB_RUN_ID}" -q | xargs -r sudo docker rm -f
 
       - name: Start ragflow:nightly
         run: |
-          sudo DOC_ENGINE=infinity docker compose -f docker/docker-compose.yml up -d
+          sed -i '1i DOC_ENGINE=infinity' docker/.env
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} up -d
 
       - name: Run sdk tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
-            export HTTP_API_TEST_LEVEL=p3
-          else
-            export HTTP_API_TEST_LEVEL=p2
-          fi
-          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python && DOC_ENGINE=infinity uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
+          source .venv/bin/activate && DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
 
       - name: Run frontend api tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && UV_LINK_MODE=copy uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
+          source .venv/bin/activate && DOC_ENGINE=infinity pytest -s --tb=short sdk/python/test/test_frontend_api/get_email.py sdk/python/test/test_frontend_api/test_dataset.py
 
       - name: Run http api tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
-            export HTTP_API_TEST_LEVEL=p3
-          else
-            export HTTP_API_TEST_LEVEL=p2
-          fi
-          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && DOC_ENGINE=infinity uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
+          source .venv/bin/activate && DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
         run: |
-          sudo DOC_ENGINE=infinity docker compose -f docker/docker-compose.yml down -v
+          # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v || true
+          sudo docker ps -a --filter "label=com.docker.compose.project=${GITHUB_RUN_ID}" -q | xargs -r sudo docker rm -f
+          if [[ -n ${RAGFLOW_IMAGE} ]]; then
+            sudo docker rmi -f ${RAGFLOW_IMAGE}
+          fi

.gitignore

@@ -149,7 +149,7 @@ out
 # Nuxt.js build / generate output
 .nuxt
 dist
-
+ragflow_cli.egg-info
 # Gatsby files
 .cache/
 # Comment in the public line in if your project uses Gatsby and not Next.js

CLAUDE.md

@@ -0,0 +1,116 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It's a full-stack application with:
+- Python backend (Flask-based API server)
+- React/TypeScript frontend (built with UmiJS)
+- Microservices architecture with Docker deployment
+- Multiple data stores (MySQL, Elasticsearch/Infinity, Redis, MinIO)
+
+## Architecture
+
+### Backend (`/api/`)
+- **Main Server**: `api/ragflow_server.py` - Flask application entry point
+- **Apps**: Modular Flask blueprints in `api/apps/` for different functionalities:
+  - `kb_app.py` - Knowledge base management
+  - `dialog_app.py` - Chat/conversation handling
+  - `document_app.py` - Document processing
+  - `canvas_app.py` - Agent workflow canvas
+  - `file_app.py` - File upload/management
+- **Services**: Business logic in `api/db/services/`
+- **Models**: Database models in `api/db/db_models.py`
+
+### Core Processing (`/rag/`)
+- **Document Processing**: `deepdoc/` - PDF parsing, OCR, layout analysis
+- **LLM Integration**: `rag/llm/` - Model abstractions for chat, embedding, reranking
+- **RAG Pipeline**: `rag/flow/` - Chunking, parsing, tokenization
+- **Graph RAG**: `graphrag/` - Knowledge graph construction and querying
+
+### Agent System (`/agent/`)
+- **Components**: Modular workflow components (LLM, retrieval, categorize, etc.)
+- **Templates**: Pre-built agent workflows in `agent/templates/`
+- **Tools**: External API integrations (Tavily, Wikipedia, SQL execution, etc.)
+
+### Frontend (`/web/`)
+- React/TypeScript with UmiJS framework
+- Ant Design + shadcn/ui components
+- State management with Zustand
+- Tailwind CSS for styling
+
+## Common Development Commands
+
+### Backend Development
+```bash
+# Install Python dependencies
+uv sync --python 3.10 --all-extras
+uv run download_deps.py
+pre-commit install
+
+# Start dependent services
+docker compose -f docker/docker-compose-base.yml up -d
+
+# Run backend (requires services to be running)
+source .venv/bin/activate
+export PYTHONPATH=$(pwd)
+bash docker/launch_backend_service.sh
+
+# Run tests
+uv run pytest
+
+# Linting
+ruff check
+ruff format
+```
+
+### Frontend Development
+```bash
+cd web
+npm install
+npm run dev        # Development server
+npm run build      # Production build
+npm run lint       # ESLint
+npm run test       # Jest tests
+```
+
+### Docker Development
+```bash
+# Full stack with Docker
+cd docker
+docker compose -f docker-compose.yml up -d
+
+# Check server status
+docker logs -f ragflow-server
+
+# Rebuild images
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
+```
+
+## Key Configuration Files
+
+- `docker/.env` - Environment variables for Docker deployment
+- `docker/service_conf.yaml.template` - Backend service configuration
+- `pyproject.toml` - Python dependencies and project configuration
+- `web/package.json` - Frontend dependencies and scripts
+
+## Testing
+
+- **Python**: pytest with markers (p1/p2/p3 priority levels)
+- **Frontend**: Jest with React Testing Library
+- **API Tests**: HTTP API and SDK tests in `test/` and `sdk/python/test/`
+
+## Database Engines
+
+RAGFlow supports switching between Elasticsearch (default) and Infinity:
+- Set `DOC_ENGINE=infinity` in `docker/.env` to use Infinity
+- Requires container restart: `docker compose down -v && docker compose up -d`
+
+## Development Environment Requirements
+
+- Python 3.10-3.12
+- Node.js >=18.20.4
+- Docker & Docker Compose
+- uv package manager
+- 16GB+ RAM, 50GB+ disk space

Dockerfile

@@ -4,8 +4,6 @@ USER root
 SHELL ["/bin/bash", "-c"]
 
 ARG NEED_MIRROR=0
-ARG LIGHTEN=0
-ENV LIGHTEN=${LIGHTEN}
 
 WORKDIR /ragflow
 
@@ -17,13 +15,6 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co
         /huggingface.co/InfiniFlow/text_concat_xgb_v1.0 \
         /huggingface.co/InfiniFlow/deepdoc \
         | tar -xf - --strip-components=3 -C /ragflow/rag/res/deepdoc 
-RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co,target=/huggingface.co \
-    if [ "$LIGHTEN" != "1" ]; then \
-        (tar -cf - \
-            /huggingface.co/BAAI/bge-large-zh-v1.5 \
-            /huggingface.co/maidalun1020/bce-embedding-base_v1 \
-            | tar -xf - --strip-components=2 -C /root/.ragflow) \
-    fi
 
 # https://github.com/chrismattmann/tika-python
 # This is the only way to run python-tika without internet access. Without this set, the default is to check the tika version and pull latest every time from Apache.
@@ -63,11 +54,11 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     apt install -y ghostscript
 
 RUN if [ "$NEED_MIRROR" == "1" ]; then \
-        pip3 config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
-        pip3 config set global.trusted-host mirrors.aliyun.com; \
+        pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \
+        pip3 config set global.trusted-host pypi.tuna.tsinghua.edu.cn; \
         mkdir -p /etc/uv && \
         echo "[[index]]" > /etc/uv/uv.toml && \
-        echo 'url = "https://mirrors.aliyun.com/pypi/simple"' >> /etc/uv/uv.toml && \
+        echo 'url = "https://pypi.tuna.tsinghua.edu.cn/simple"' >> /etc/uv/uv.toml && \
         echo "default = true" >> /etc/uv/uv.toml; \
     fi; \
     pipx install uv
@@ -151,15 +142,11 @@ COPY pyproject.toml uv.lock ./
 # uv records index url into uv.lock but doesn't failover among multiple indexes
 RUN --mount=type=cache,id=ragflow_uv,target=/root/.cache/uv,sharing=locked \
     if [ "$NEED_MIRROR" == "1" ]; then \
-        sed -i 's|pypi.org|mirrors.aliyun.com/pypi|g' uv.lock; \
+        sed -i 's|pypi.org|pypi.tuna.tsinghua.edu.cn|g' uv.lock; \
     else \
-        sed -i 's|mirrors.aliyun.com/pypi|pypi.org|g' uv.lock; \
+        sed -i 's|pypi.tuna.tsinghua.edu.cn|pypi.org|g' uv.lock; \
     fi; \
-    if [ "$LIGHTEN" == "1" ]; then \
-        uv sync --python 3.10 --frozen; \
-    else \
-        uv sync --python 3.10 --frozen --all-extras; \
-    fi
+    uv sync --python 3.10 --frozen
 
 COPY web web
 COPY docs docs
@@ -169,11 +156,7 @@ RUN --mount=type=cache,id=ragflow_npm,target=/root/.npm,sharing=locked \
 COPY .git /ragflow/.git
 
 RUN version_info=$(git describe --tags --match=v* --first-parent --always); \
-    if [ "$LIGHTEN" == "1" ]; then \
-        version_info="$version_info slim"; \
-    else \
-        version_info="$version_info full"; \
-    fi; \
+    version_info="$version_info"; \
     echo "RAGFlow version: $version_info"; \
     echo $version_info > /ragflow/VERSION
 
@@ -191,6 +174,7 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 ENV PYTHONPATH=/ragflow/
 
 COPY web web
+COPY admin admin
 COPY api api
 COPY conf conf
 COPY deepdoc deepdoc
@@ -201,6 +185,7 @@ COPY agentic_reasoning agentic_reasoning
 COPY pyproject.toml uv.lock ./
 COPY mcp mcp
 COPY plugin plugin
+COPY common common
 
 COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template
 COPY docker/entrypoint.sh ./

Dockerfile_tei

@@ -0,0 +1,14 @@
+FROM ghcr.io/huggingface/text-embeddings-inference:cpu-1.8
+
+# uv tool install huggingface_hub
+# hf download --local-dir tei_data/BAAI/bge-small-en-v1.5 BAAI/bge-small-en-v1.5
+# hf download --local-dir tei_data/BAAI/bge-m3 BAAI/bge-m3
+# hf download --local-dir tei_data/Qwen/Qwen3-Embedding-0.6B Qwen/Qwen3-Embedding-0.6B
+COPY tei_data /data
+
+# curl -X POST http://localhost:6380/embed -H "Content-Type: application/json" -d '{"inputs": "Hello, world! This is a test sentence."}'
+# curl -X POST http://tei:80/embed -H "Content-Type: application/json" -d '{"inputs": "Hello, world! This is a test sentence."}'
+# [[-0.058816575,0.019564206,0.026697718,...]]
+
+# curl -X POST http://localhost:6380/v1/embeddings -H "Content-Type: application/json" -d '{"input": "Hello, world! This is a test sentence."}'
+# {"object":"list","data":[{"object":"embedding","embedding":[-0.058816575,0.019564206,...],"index":0}],"model":"BAAI/bge-small-en-v1.5","usage":{"prompt_tokens":12,"total_tokens":12}}

README.md

@@ -1,6 +1,6 @@
 <div align="center">
 <a href="https://demo.ragflow.io/">
-<img src="web/src/assets/logo-with-text.png" width="520" alt="ragflow logo">
+<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
 </a>
 </div>
 
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.20.5">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -43,7 +43,9 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
 
 <div align="center">
 <a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
@@ -59,8 +61,7 @@
 - 🔎 [System Architecture](#-system-architecture)
 - 🎬 [Get Started](#-get-started)
 - 🔧 [Configurations](#-configurations)
-- 🔧 [Build a docker image without embedding models](#-build-a-docker-image-without-embedding-models)
-- 🔧 [Build a docker image including embedding models](#-build-a-docker-image-including-embedding-models)
+- 🔧 [Build a Docker image](#-build-a-docker-image)
 - 🔨 [Launch service from source for development](#-launch-service-from-source-for-development)
 - 📚 [Documentation](#-documentation)
 - 📜 [Roadmap](#-roadmap)
@@ -84,13 +85,14 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Latest Updates
 
+- 2025-11-12 Supports data synchronization from Confluence, AWS S3, Discord, Google Drive.
+- 2025-10-23 Supports MinerU & Docling as document parsing methods.
+- 2025-10-15 Supports orchestrable ingestion pipeline.
 - 2025-08-08 Supports OpenAI's latest GPT-5 series models.
-- 2025-08-04 Supports new models, including Kimi K2 and Grok 4.
 - 2025-08-01 Supports agentic workflow and MCP.
 - 2025-05-23 Adds a Python/JavaScript code executor component to Agent.
 - 2025-05-05 Supports cross-language query.
 - 2025-03-19 Supports using a multi-modal model to make sense of images within PDF or DOCX files.
-- 2025-02-28 Combined with Internet search (Tavily), supports reasoning like Deep Research for any LLMs.
 - 2024-12-18 Upgrades Document Layout Analysis model in DeepDoc.
 - 2024-08-22 Support text to SQL statements through RAG.
 
@@ -135,7 +137,7 @@ releases! 🌟
 ## 🔎 System Architecture
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+<img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
 ## 🎬 Get Started
@@ -174,41 +176,46 @@ releases! 🌟
    > ```bash
    > vm.max_map_count=262144
    > ```
-
+   >
 2. Clone the repo:
 
    ```bash
    $ git clone https://github.com/infiniflow/ragflow.git
    ```
-
 3. Start up the server using the pre-built Docker images:
 
 > [!CAUTION]
 > All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
 > If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
 
-   > The command below downloads the `v0.20.5-slim` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.20.5-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.20.5` for the full edition `v0.20.5`.
+> The command below downloads the `v0.22.0` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.22.0`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
 
-   ```bash
+```bash
    $ cd ragflow/docker
-   # Use CPU for embedding and DeepDoc tasks:
+   
+   # Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases), e.g.: git checkout v0.22.0
+
+   # Use CPU for DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
 
-   # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
-   ```
+   # To use GPU to accelerate DeepDoc tasks:
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
+```
 
-   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-   |-------------------|-----------------|-----------------------|--------------------------|
-   | v0.20.5           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.20.5-slim      | &approx;2       | ❌                   | Stable release            |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                   | _Unstable_ nightly build  |
+> Note: Prior to `v0.22.0`, we provided both images with embedding models and slim images without embedding models. Details as follows:
+
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+| ----------------- | --------------- | --------------------- | ------------------------ |
+| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+
+> Starting with `v0.22.0`, we ship only the slim edition and no longer append the **-slim** suffix to the image tag.
 
 4. Check the server status after having the server up and running:
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _The following output confirms a successful launch of the system:_
@@ -226,14 +233,17 @@ releases! 🌟
 
    > If you skip this confirmation step and directly log in to RAGFlow, your browser may prompt a `network anormal`
    > error because, at that moment, your RAGFlow may not be fully initialized.
-
+   >
 5. In your web browser, enter the IP address of your server and log in to RAGFlow.
+
    > With the default settings, you only need to enter `http://IP_OF_YOUR_MACHINE` (**sans** port number) as the default
    > HTTP serving port `80` can be omitted when using the default configurations.
+   >
 6. In [service_conf.yaml.template](./docker/service_conf.yaml.template), select the desired LLM factory in `user_default_llm` and update
    the `API_KEY` field with the corresponding API key.
 
    > See [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) for more information.
+   >
 
    _The show is on!_
 
@@ -272,7 +282,6 @@ RAGFlow uses Elasticsearch by default for storing full text and vectors. To swit
 > `-v` will delete the docker container volumes, and the existing data will be cleared.
 
 2. Set `DOC_ENGINE` in **docker/.env** to `infinity`.
-
 3. Start the containers:
 
    ```bash
@@ -282,20 +291,10 @@ RAGFlow uses Elasticsearch by default for storing full text and vectors. To swit
 > [!WARNING]
 > Switching to Infinity on a Linux/arm64 machine is not yet officially supported.
 
-## 🔧 Build a Docker image without embedding models
+## 🔧 Build a Docker image
 
 This image is approximately 2 GB in size and relies on external LLM and embedding services.
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 Build a Docker image including embedding models
-
-This image is approximately 9 GB in size. As it includes embedding models, it relies on external LLM services only.
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -309,17 +308,15 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    pipx install uv pre-commit
    ```
-
 2. Clone the source code and install Python dependencies:
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```
-
 3. Launch the dependent services (MinIO, Elasticsearch, Redis, and MySQL) using Docker Compose:
 
    ```bash
@@ -331,24 +328,23 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. If you cannot access HuggingFace, set the `HF_ENDPOINT` environment variable to use a mirror site:
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. If your operating system does not have jemalloc, please install it as follows:
 
    ```bash
-   # ubuntu
+   # Ubuntu
    sudo apt-get install libjemalloc-dev
-   # centos
+   # CentOS
    sudo yum install jemalloc
-   # mac
+   # OpenSUSE
+   sudo zypper install jemalloc
+   # macOS
    sudo brew install jemalloc
    ```
-
 6. Launch backend service:
 
    ```bash
@@ -356,14 +352,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. Install frontend dependencies:
 
    ```bash
    cd web
    npm install
    ```
-
 8. Launch frontend service:
 
    ```bash
@@ -373,14 +367,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    _The following output confirms a successful launch of the system:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
-
 9. Stop RAGFlow front-end and back-end service after development is complete:
 
    ```bash
    pkill -f "ragflow_server.py|task_executor.py"
    ```
 
-
 ## 📚 Documentation
 
 - [Quickstart](https://ragflow.io/docs/dev/)

README_id.md

@@ -1,6 +1,6 @@
 <div align="center">
 <a href="https://demo.ragflow.io/">
-<img src="web/src/assets/logo-with-text.png" width="520" alt="Logo ragflow">
+<img src="web/src/assets/logo-with-text.svg" width="520" alt="Logo ragflow">
 </a>
 </div>
 
@@ -22,7 +22,7 @@
         <img alt="Lencana Daring" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.20.5">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Rilis%20Terbaru" alt="Rilis Terbaru">
@@ -43,7 +43,13 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
 
 <details open>
 <summary><b>📕 Daftar Isi </b> </summary>
@@ -55,8 +61,7 @@
 - 🔎 [Arsitektur Sistem](#-arsitektur-sistem)
 - 🎬 [Mulai](#-mulai)
 - 🔧 [Konfigurasi](#-konfigurasi)
-- 🔧 [Membangun Image Docker tanpa Model Embedding](#-membangun-image-docker-tanpa-model-embedding)
-- 🔧 [Membangun Image Docker dengan Model Embedding](#-membangun-image-docker-dengan-model-embedding)
+- 🔧 [Membangun Image Docker](#-membangun-docker-image)
 - 🔨 [Meluncurkan aplikasi dari Sumber untuk Pengembangan](#-meluncurkan-aplikasi-dari-sumber-untuk-pengembangan)
 - 📚 [Dokumentasi](#-dokumentasi)
 - 📜 [Peta Jalan](#-peta-jalan)
@@ -80,13 +85,14 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Pembaruan Terbaru
 
+- 2025-11-12 Mendukung sinkronisasi data dari Confluence, AWS S3, Discord, Google Drive.
+- 2025-10-23 Mendukung MinerU & Docling sebagai metode penguraian dokumen.
+- 2025-10-15 Dukungan untuk jalur data yang terorkestrasi.
 - 2025-08-08 Mendukung model seri GPT-5 terbaru dari OpenAI.
-- 2025-08-04 Mendukung model baru, termasuk Kimi K2 dan Grok 4.
 - 2025-08-01 Mendukung alur kerja agen dan MCP.
 - 2025-05-23 Menambahkan komponen pelaksana kode Python/JS ke Agen.
 - 2025-05-05 Mendukung kueri lintas bahasa.
 - 2025-03-19 Mendukung penggunaan model multi-modal untuk memahami gambar di dalam file PDF atau DOCX.
-- 2025-02-28 dikombinasikan dengan pencarian Internet (TAVILY), mendukung penelitian mendalam untuk LLM apa pun.
 - 2024-12-18 Meningkatkan model Analisis Tata Letak Dokumen di DeepDoc.
 - 2024-08-22 Dukungan untuk teks ke pernyataan SQL melalui RAG.
 
@@ -129,7 +135,7 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 ## 🔎 Arsitektur Sistem
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+<img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
 ## 🎬 Mulai
@@ -168,41 +174,46 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
    > ```bash
    > vm.max_map_count=262144
    > ```
-
+   >
 2. Clone repositori:
 
    ```bash
    $ git clone https://github.com/infiniflow/ragflow.git
    ```
-
 3. Bangun image Docker pre-built dan jalankan server:
 
 > [!CAUTION]
 > Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
 > Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
 
-> Perintah di bawah ini mengunduh edisi v0.20.5-slim dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.20.5-slim, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server. Misalnya, atur RAGFLOW_IMAGE=infiniflow/ragflow:v0.20.5 untuk edisi lengkap v0.20.5.
+> Perintah di bawah ini mengunduh edisi v0.22.0 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.22.0, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
 
 ```bash
-$ cd ragflow/docker
-# Use CPU for embedding and DeepDoc tasks:
-$ docker compose -f docker-compose.yml up -d
+   $ cd ragflow/docker
+   
+   # Opsional: gunakan tag stabil (lihat releases: https://github.com/infiniflow/ragflow/releases), contoh: git checkout v0.22.0
 
-# To use GPU to accelerate embedding and DeepDoc tasks:
-# docker compose -f docker-compose-gpu.yml up -d
+   # Use CPU for DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
+
+   # To use GPU to accelerate DeepDoc tasks:
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
 ```
 
+> Catatan: Sebelum `v0.22.0`, kami menyediakan image dengan model embedding dan image slim tanpa model embedding. Detailnya sebagai berikut:
+
 | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
 | ----------------- | --------------- | --------------------- | ------------------------ |
-| v0.20.5           | &approx;9       | :heavy_check_mark:    | Stable release           |
-| v0.20.5-slim      | &approx;2       | ❌                    | Stable release           |
-| nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-| nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
+| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+
+> Mulai dari `v0.22.0`, kami hanya menyediakan edisi slim dan tidak lagi menambahkan akhiran **-slim** pada tag image.
 
 1. Periksa status server setelah server aktif dan berjalan:
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _Output berikut menandakan bahwa sistem berhasil diluncurkan:_
@@ -220,14 +231,17 @@ $ docker compose -f docker-compose.yml up -d
 
    > Jika Anda melewatkan langkah ini dan langsung login ke RAGFlow, browser Anda mungkin menampilkan error `network anormal`
    > karena RAGFlow mungkin belum sepenuhnya siap.
-
+   >
 2. Buka browser web Anda, masukkan alamat IP server Anda, dan login ke RAGFlow.
+
    > Dengan pengaturan default, Anda hanya perlu memasukkan `http://IP_DEVICE_ANDA` (**tanpa** nomor port) karena
    > port HTTP default `80` bisa dihilangkan saat menggunakan konfigurasi default.
+   >
 3. Dalam [service_conf.yaml.template](./docker/service_conf.yaml.template), pilih LLM factory yang diinginkan di `user_default_llm` dan perbarui
    bidang `API_KEY` dengan kunci API yang sesuai.
 
    > Lihat [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) untuk informasi lebih lanjut.
+   >
 
    _Sistem telah siap digunakan!_
 
@@ -249,20 +263,10 @@ Pembaruan konfigurasi ini memerlukan reboot semua kontainer agar efektif:
 > $ docker compose -f docker-compose.yml up -d
 > ```
 
-## 🔧 Membangun Docker Image tanpa Model Embedding
+## 🔧 Membangun Docker Image
 
 Image ini berukuran sekitar 2 GB dan bergantung pada aplikasi LLM eksternal dan embedding.
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 Membangun Docker Image Termasuk Model Embedding
-
-Image ini berukuran sekitar 9 GB. Karena sudah termasuk model embedding, ia hanya bergantung pada aplikasi LLM eksternal.
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -276,17 +280,15 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    pipx install uv pre-commit
    ```
-
 2. Clone kode sumber dan instal dependensi Python:
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```
-
 3. Jalankan aplikasi yang diperlukan (MinIO, Elasticsearch, Redis, dan MySQL) menggunakan Docker Compose:
 
    ```bash
@@ -298,13 +300,11 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. Jika Anda tidak dapat mengakses HuggingFace, atur variabel lingkungan `HF_ENDPOINT` untuk menggunakan situs mirror:
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. Jika sistem operasi Anda tidak memiliki jemalloc, instal sebagai berikut:
 
    ```bash
@@ -315,7 +315,6 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    # mac
    sudo brew install jemalloc
    ```
-
 6. Jalankan aplikasi backend:
 
    ```bash
@@ -323,14 +322,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. Instal dependensi frontend:
 
    ```bash
    cd web
    npm install
    ```
-
 8. Jalankan aplikasi frontend:
 
    ```bash
@@ -340,15 +337,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    _Output berikut menandakan bahwa sistem berhasil diluncurkan:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
-
-
 9. Hentikan layanan front-end dan back-end RAGFlow setelah pengembangan selesai:
 
    ```bash
    pkill -f "ragflow_server.py|task_executor.py"
    ```
 
-
 ## 📚 Dokumentasi
 
 - [Quickstart](https://ragflow.io/docs/dev/)

README_ja.md

@@ -1,6 +1,6 @@
 <div align="center">
 <a href="https://demo.ragflow.io/">
-<img src="web/src/assets/logo-with-text.png" width="350" alt="ragflow logo">
+<img src="web/src/assets/logo-with-text.svg" width="350" alt="ragflow logo">
 </a>
 </div>
 
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.20.5">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -43,7 +43,13 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
 
 ## 💡 RAGFlow とは？
 
@@ -60,13 +66,14 @@
 
 ## 🔥 最新情報
 
+- 2025-11-12 Confluence、AWS S3、Discord、Google Drive からのデータ同期をサポートします。
+- 2025-10-23 ドキュメント解析方法として MinerU と Docling をサポートします。
+- 2025-10-15 オーケストレーションされたデータパイプラインのサポート。
 - 2025-08-08 OpenAI の最新 GPT-5 シリーズモデルをサポートします。
-- 2025-08-04 新モデル、キミK2およびGrok 4をサポート。
 - 2025-08-01 エージェントワークフローとMCPをサポート。
 - 2025-05-23 エージェントに Python/JS コードエグゼキュータコンポーネントを追加しました。
 - 2025-05-05 言語間クエリをサポートしました。
 - 2025-03-19 PDFまたはDOCXファイル内の画像を理解するために、多モーダルモデルを使用することをサポートします。
-- 2025-02-28 インターネット検索 (TAVILY) と組み合わせて、あらゆる LLM の詳細な調査をサポートします。
 - 2024-12-18 DeepDoc のドキュメント レイアウト分析モデルをアップグレードします。
 - 2024-08-22 RAG を介して SQL ステートメントへのテキストをサポートします。
 
@@ -109,7 +116,7 @@
 ## 🔎 システム構成
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+<img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
 ## 🎬 初期設定
@@ -147,41 +154,46 @@
    > ```bash
    > vm.max_map_count=262144
    > ```
-
+   >
 2. リポジトリをクローンする:
 
    ```bash
    $ git clone https://github.com/infiniflow/ragflow.git
    ```
-
 3. ビルド済みの Docker イメージをビルドし、サーバーを起動する:
 
 > [!CAUTION]
 > 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
 > ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
 
-   > 以下のコマンドは、RAGFlow Docker イメージの v0.20.5-slim エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.20.5-slim とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。例えば、完全版 v0.20.5 をダウンロードするには、RAGFLOW_IMAGE=infiniflow/ragflow:v0.20.5 と設定します。
+> 以下のコマンドは、RAGFlow Docker イメージの v0.22.0 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.22.0 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
 
-   ```bash
+```bash
    $ cd ragflow/docker
-   # Use CPU for embedding and DeepDoc tasks:
+
+   # 任意: 安定版タグを利用 (一覧: https://github.com/infiniflow/ragflow/releases) 例: git checkout v0.22.0
+
+   # Use CPU for DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
 
-   # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
-   ```
+   # To use GPU to accelerate DeepDoc tasks:
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
+```
 
-   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-   | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.20.5           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.20.5-slim      | &approx;2       | ❌                    | Stable release           |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+> 注意：`v0.22.0` より前のバージョンでは、embedding モデルを含むイメージと、embedding モデルを含まない slim イメージの両方を提供していました。詳細は以下の通りです：
 
-1. サーバーを立ち上げた後、サーバーの状態を確認する:
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+| ----------------- | --------------- | --------------------- | ------------------------ |
+| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
 
+> `v0.22.0` 以降、当プロジェクトでは slim エディションのみを提供し、イメージタグに **-slim** サフィックスを付けなくなりました。
+
+   1. サーバーを立ち上げた後、サーバーの状態を確認する:   
+   
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _以下の出力は、システムが正常に起動したことを確認するものです:_
@@ -197,12 +209,15 @@
    ```
 
    > もし確認ステップをスキップして直接 RAGFlow にログインした場合、その時点で RAGFlow が完全に初期化されていない可能性があるため、ブラウザーがネットワーク異常エラーを表示するかもしれません。
-
+   >
 2. ウェブブラウザで、プロンプトに従ってサーバーの IP アドレスを入力し、RAGFlow にログインします。
+
    > デフォルトの設定を使用する場合、デフォルトの HTTP サービングポート `80` は省略できるので、与えられたシナリオでは、`http://IP_OF_YOUR_MACHINE`（ポート番号は省略）だけを入力すればよい。
+   >
 3. [service_conf.yaml.template](./docker/service_conf.yaml.template) で、`user_default_llm` で希望の LLM ファクトリを選択し、`API_KEY` フィールドを対応する API キーで更新する。
 
    > 詳しくは [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) を参照してください。
+   >
 
    _これで初期設定完了！ショーの開幕です！_
 
@@ -231,33 +246,27 @@
 RAGFlow はデフォルトで Elasticsearch を使用して全文とベクトルを保存します。［Infinity］に切り替え（https://github.com/infiniflow/infinity/)、次の手順に従います。
 
 1. 実行中のすべてのコンテナを停止するには：
+
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+
    Note: `-v` は docker コンテナのボリュームを削除し、既存のデータをクリアします。
 2. **docker/.env** の「DOC \_ ENGINE」を「infinity」に設定します。
-
 3. 起動コンテナ：
+
    ```bash
    $ docker compose -f docker-compose.yml up -d
    ```
+
    > [!WARNING]
    > Linux/arm64 マシンでの Infinity への切り替えは正式にサポートされていません。
+   >
 
-## 🔧 ソースコードで Docker イメージを作成（埋め込みモデルなし）
+## 🔧 ソースコードで Docker イメージを作成
 
 この Docker イメージのサイズは約 1GB で、外部の大モデルと埋め込みサービスに依存しています。
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 ソースコードをコンパイルした Docker イメージ（埋め込みモデルを含む）
-
-この Docker のサイズは約 9GB で、埋め込みモデルを含むため、外部の大モデルサービスのみが必要です。
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -271,17 +280,15 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    pipx install uv pre-commit
    ```
-
 2. ソースコードをクローンし、Python の依存関係をインストールする:
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```
-
 3. Docker Compose を使用して依存サービス（MinIO、Elasticsearch、Redis、MySQL）を起動する:
 
    ```bash
@@ -293,13 +300,11 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. HuggingFace にアクセスできない場合は、`HF_ENDPOINT` 環境変数を設定してミラーサイトを使用してください:
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. オペレーティングシステムにjemallocがない場合は、次のようにインストールします:
 
    ```bash
@@ -310,7 +315,6 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    # mac
    sudo brew install jemalloc
    ```
-
 6. バックエンドサービスを起動する:
 
    ```bash
@@ -318,14 +322,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. フロントエンドの依存関係をインストールする:
 
    ```bash
    cd web
    npm install
    ```
-
 8. フロントエンドサービスを起動する:
 
    ```bash
@@ -335,14 +337,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    _以下の画面で、システムが正常に起動したことを示します:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
-
 9. 開発が完了したら、RAGFlow のフロントエンド サービスとバックエンド サービスを停止します:
 
    ```bash
    pkill -f "ragflow_server.py|task_executor.py"
    ```
 
-
 ## 📚 ドキュメンテーション
 
 - [Quickstart](https://ragflow.io/docs/dev/)

README_ko.md

@@ -1,6 +1,6 @@
 <div align="center">
 <a href="https://demo.ragflow.io/">
-<img src="web/src/assets/logo-with-text.png" width="520" alt="ragflow logo">
+<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
 </a>
 </div>
 
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.20.5">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -43,7 +43,14 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
+
 
 ## 💡 RAGFlow란?
 
@@ -60,13 +67,14 @@
 
 ## 🔥 업데이트
 
+- 2025-11-12 Confluence, AWS S3, Discord, Google Drive에서 데이터 동기화를 지원합니다.
+- 2025-10-23 문서 파싱 방법으로 MinerU 및 Docling을 지원합니다.
+- 2025-10-15 조정된 데이터 파이프라인 지원.
 - 2025-08-08 OpenAI의 최신 GPT-5 시리즈 모델을 지원합니다.
-- 2025-08-04 새로운 모델인 Kimi K2와 Grok 4를 포함하여 지원합니다.
 - 2025-08-01 에이전트 워크플로우와 MCP를 지원합니다.
 - 2025-05-23 Agent에 Python/JS 코드 실행기 구성 요소를 추가합니다.
 - 2025-05-05 언어 간 쿼리를 지원합니다.
 - 2025-03-19 PDF 또는 DOCX 파일 내의 이미지를 이해하기 위해 다중 모드 모델을 사용하는 것을 지원합니다.
-- 2025-02-28 인터넷 검색(TAVILY)과 결합되어 모든 LLM에 대한 심층 연구를 지원합니다.
 - 2024-12-18 DeepDoc의 문서 레이아웃 분석 모델 업그레이드.
 - 2024-08-22 RAG를 통해 SQL 문에 텍스트를 지원합니다.
 
@@ -109,7 +117,7 @@
 ## 🔎 시스템 아키텍처
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+<img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
 ## 🎬 시작하기
@@ -160,28 +168,34 @@
 > 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
 > ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
 
-   > 아래 명령어는 RAGFlow Docker 이미지의 v0.20.5-slim 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.20.5-slim과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오. 예를 들어, 전체 버전인 v0.20.5을 다운로드하려면 RAGFLOW_IMAGE=infiniflow/ragflow:v0.20.5로 설정합니다.
+   > 아래 명령어는 RAGFlow Docker 이미지의 v0.22.0 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.22.0과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
 
    ```bash
    $ cd ragflow/docker
-   # Use CPU for embedding and DeepDoc tasks:
+   
+   # Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases), e.g.: git checkout v0.22.0
+
+   # Use CPU for DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
 
-   # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
-   ```
+   # To use GPU to accelerate DeepDoc tasks:
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
+```
 
-   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-   | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.20.5           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.20.5-slim      | &approx;2       | ❌                    | Stable release           |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+> 참고: `v0.22.0` 이전 버전에서는 embedding 모델이 포함된 이미지와 embedding 모델이 포함되지 않은 slim 이미지를 모두 제공했습니다. 자세한 내용은 다음과 같습니다:
+
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+| ----------------- | --------------- | --------------------- | ------------------------ |
+| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+
+> `v0.22.0`부터는 slim 에디션만 배포하며 이미지 태그에 **-slim** 접미사를 더 이상 붙이지 않습니다.
 
 1. 서버가 시작된 후 서버 상태를 확인하세요:
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _다음 출력 결과로 시스템이 성공적으로 시작되었음을 확인합니다:_
@@ -243,20 +257,10 @@ RAGFlow 는 기본적으로 Elasticsearch 를 사용하여 전체 텍스트 및
    > [!WARNING]
    > Linux/arm64 시스템에서 Infinity로 전환하는 것은 공식적으로 지원되지 않습니다.
 
-## 🔧 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함하지 않음)
+## 🔧 소스 코드로 Docker 이미지를 컴파일합니다
 
 이 Docker 이미지의 크기는 약 1GB이며, 외부 대형 모델과 임베딩 서비스에 의존합니다.
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함)
-
-이 Docker의 크기는 약 9GB이며, 이미 임베딩 모델을 포함하고 있으므로 외부 대형 모델 서비스에만 의존하면 됩니다.
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -276,7 +280,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

README_pt_br.md

@@ -1,6 +1,6 @@
 <div align="center">
 <a href="https://demo.ragflow.io/">
-<img src="web/src/assets/logo-with-text.png" width="520" alt="ragflow logo">
+<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
 </a>
 </div>
 
@@ -22,7 +22,7 @@
         <img alt="Badge Estático" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.20.5">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Última%20Relese" alt="Última Versão">
@@ -43,7 +43,13 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
 
 <details open>
 <summary><b>📕 Índice</b></summary>
@@ -80,13 +86,14 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Últimas Atualizações
 
+- 12-11-2025 Suporta a sincronização de dados do Confluence, AWS S3, Discord e Google Drive.
+- 23-10-2025 Suporta MinerU e Docling como métodos de análise de documentos.
+- 15-10-2025 Suporte para pipelines de dados orquestrados.
 - 08-08-2025 Suporta a mais recente série GPT-5 da OpenAI.
-- 04-08-2025 Suporta novos modelos, incluindo Kimi K2 e Grok 4.
 - 01-08-2025 Suporta fluxo de trabalho agente e MCP.
 - 23-05-2025 Adicione o componente executor de código Python/JS ao Agente.
 - 05-05-2025 Suporte a consultas entre idiomas.
 - 19-03-2025 Suporta o uso de um modelo multi-modal para entender imagens dentro de arquivos PDF ou DOCX.
-- 28-02-2025 combinado com a pesquisa na Internet (T AVI LY), suporta pesquisas profundas para qualquer LLM.
 - 18-12-2024 Atualiza o modelo de Análise de Layout de Documentos no DeepDoc.
 - 22-08-2024 Suporta conversão de texto para comandos SQL via RAG.
 
@@ -129,7 +136,7 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 ## 🔎 Arquitetura do Sistema
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+<img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
 ## 🎬 Primeiros Passos
@@ -147,84 +154,90 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ### 🚀 Iniciar o servidor
 
-1.  Certifique-se de que `vm.max_map_count` >= 262144:
+1. Certifique-se de que `vm.max_map_count` >= 262144:
 
-    > Para verificar o valor de `vm.max_map_count`:
-    >
-    > ```bash
-    > $ sysctl vm.max_map_count
-    > ```
-    >
-    > Se necessário, redefina `vm.max_map_count` para um valor de pelo menos 262144:
-    >
-    > ```bash
-    > # Neste caso, defina para 262144:
-    > $ sudo sysctl -w vm.max_map_count=262144
-    > ```
-    >
-    > Essa mudança será resetada após a reinicialização do sistema. Para garantir que a alteração permaneça permanente, adicione ou atualize o valor de `vm.max_map_count` em **/etc/sysctl.conf**:
-    >
-    > ```bash
-    > vm.max_map_count=262144
-    > ```
+   > Para verificar o valor de `vm.max_map_count`:
+   >
+   > ```bash
+   > $ sysctl vm.max_map_count
+   > ```
+   >
+   > Se necessário, redefina `vm.max_map_count` para um valor de pelo menos 262144:
+   >
+   > ```bash
+   > # Neste caso, defina para 262144:
+   > $ sudo sysctl -w vm.max_map_count=262144
+   > ```
+   >
+   > Essa mudança será resetada após a reinicialização do sistema. Para garantir que a alteração permaneça permanente, adicione ou atualize o valor de `vm.max_map_count` em **/etc/sysctl.conf**:
+   >
+   > ```bash
+   > vm.max_map_count=262144
+   > ```
+   >
+2. Clone o repositório:
 
-2.  Clone o repositório:
-
-    ```bash
-    $ git clone https://github.com/infiniflow/ragflow.git
-    ```
-
-3.  Inicie o servidor usando as imagens Docker pré-compiladas:
+   ```bash
+   $ git clone https://github.com/infiniflow/ragflow.git
+   ```
+3. Inicie o servidor usando as imagens Docker pré-compiladas:
 
 > [!CAUTION]
 > Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
 > Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
 
-    > O comando abaixo baixa a edição `v0.20.5-slim` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.20.5-slim`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor. Por exemplo: defina `RAGFLOW_IMAGE=infiniflow/ragflow:v0.20.5` para a edição completa `v0.20.5`.
+    > O comando abaixo baixa a edição`v0.22.0` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.22.0`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
 
-    ```bash
-    $ cd ragflow/docker
-    # Use CPU for embedding and DeepDoc tasks:
-    $ docker compose -f docker-compose.yml up -d
+```bash
+   $ cd ragflow/docker
+   
+   # Opcional: use uma tag estável (veja releases: https://github.com/infiniflow/ragflow/releases), ex.: git checkout v0.22.0
 
-    # To use GPU to accelerate embedding and DeepDoc tasks:
-    # docker compose -f docker-compose-gpu.yml up -d
-    ```
+   # Use CPU for DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
 
-    | Tag da imagem RAGFlow | Tamanho da imagem (GB) | Possui modelos de incorporação? | Estável?                 |
-    | --------------------- | ---------------------- | ------------------------------- | ------------------------ |
-    | v0.20.5               | ~9                     | :heavy_check_mark:              | Lançamento estável       |
-    | v0.20.5-slim          | ~2                     | ❌                              | Lançamento estável       |
-    | nightly               | ~9                     | :heavy_check_mark:              | _Instável_ build noturno |
-    | nightly-slim          | ~2                     | ❌                               | _Instável_ build noturno |
+   # To use GPU to accelerate DeepDoc tasks:
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
+```
 
-4.  Verifique o status do servidor após tê-lo iniciado:
+> Nota: Antes da `v0.22.0`, fornecíamos imagens com modelos de embedding e imagens slim sem modelos de embedding. Detalhes a seguir:
 
-    ```bash
-    $ docker logs -f ragflow-server
-    ```
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+| ----------------- | --------------- | --------------------- | ------------------------ |
+| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
 
-    _O seguinte resultado confirma o lançamento bem-sucedido do sistema:_
+> A partir da `v0.22.0`, distribuímos apenas a edição slim e não adicionamos mais o sufixo **-slim** às tags das imagens.
 
-    ```bash
-         ____   ___    ______ ______ __
-        / __ \ /   |  / ____// ____// /____  _      __
-       / /_/ // /| | / / __ / /_   / // __ \| | /| / /
-      / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
-     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
+4. Verifique o status do servidor após tê-lo iniciado:
 
-     * Rodando em todos os endereços (0.0.0.0)
-    ```
+   ```bash
+   $ docker logs -f docker-ragflow-cpu-1
+   ```
 
-    > Se você pular essa etapa de confirmação e acessar diretamente o RAGFlow, seu navegador pode exibir um erro `network anormal`, pois, nesse momento, seu RAGFlow pode não estar totalmente inicializado.
+   _O seguinte resultado confirma o lançamento bem-sucedido do sistema:_
 
-5.  No seu navegador, insira o endereço IP do seu servidor e faça login no RAGFlow.
+   ```bash
+        ____   ___    ______ ______ __
+       / __ \ /   |  / ____// ____// /____  _      __
+      / /_/ // /| | / / __ / /_   / // __ \| | /| / /
+     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
+    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
-    > Com as configurações padrão, você só precisa digitar `http://IP_DO_SEU_MÁQUINA` (**sem** o número da porta), pois a porta HTTP padrão `80` pode ser omitida ao usar as configurações padrão.
+    * Rodando em todos os endereços (0.0.0.0)
+   ```
 
-6.  Em [service_conf.yaml.template](./docker/service_conf.yaml.template), selecione a fábrica LLM desejada em `user_default_llm` e atualize o campo `API_KEY` com a chave de API correspondente.
+   > Se você pular essa etapa de confirmação e acessar diretamente o RAGFlow, seu navegador pode exibir um erro `network anormal`, pois, nesse momento, seu RAGFlow pode não estar totalmente inicializado.
+   >
+5. No seu navegador, insira o endereço IP do seu servidor e faça login no RAGFlow.
 
-    > Consulte [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) para mais informações.
+   > Com as configurações padrão, você só precisa digitar `http://IP_DO_SEU_MÁQUINA` (**sem** o número da porta), pois a porta HTTP padrão `80` pode ser omitida ao usar as configurações padrão.
+   >
+6. Em [service_conf.yaml.template](./docker/service_conf.yaml.template), selecione a fábrica LLM desejada em `user_default_llm` e atualize o campo `API_KEY` com a chave de API correspondente.
+
+   > Consulte [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) para mais informações.
+   >
 
 _O show está no ar!_
 
@@ -255,9 +268,9 @@ O RAGFlow usa o Elasticsearch por padrão para armazenar texto completo e vetore
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+
    Note: `-v` irá deletar os volumes do contêiner, e os dados existentes serão apagados.
 2. Defina `DOC_ENGINE` no **docker/.env** para `infinity`.
-
 3. Inicie os contêineres:
 
    ```bash
@@ -265,22 +278,12 @@ O RAGFlow usa o Elasticsearch por padrão para armazenar texto completo e vetore
    ```
 
 > [!ATENÇÃO]
-> A mudança para o Infinity em uma máquina Linux/arm64 ainda não é oficialmente suportada.
+ > A mudança para o Infinity em uma máquina Linux/arm64 ainda não é oficialmente suportada.
 
-## 🔧 Criar uma imagem Docker sem modelos de incorporação
+## 🔧 Criar uma imagem Docker
 
 Esta imagem tem cerca de 2 GB de tamanho e depende de serviços externos de LLM e incorporação.
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 Criar uma imagem Docker incluindo modelos de incorporação
-
-Esta imagem tem cerca de 9 GB de tamanho. Como inclui modelos de incorporação, depende apenas de serviços externos de LLM.
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -294,17 +297,15 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    pipx install uv pre-commit
    ```
-
 2. Clone o código-fonte e instale as dependências Python:
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # instala os módulos Python dependentes do RAGFlow
+   uv sync --python 3.10 # instala os módulos Python dependentes do RAGFlow
    uv run download_deps.py
    pre-commit install
    ```
-
 3. Inicie os serviços dependentes (MinIO, Elasticsearch, Redis e MySQL) usando Docker Compose:
 
    ```bash
@@ -316,24 +317,21 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. Se não conseguir acessar o HuggingFace, defina a variável de ambiente `HF_ENDPOINT` para usar um site espelho:
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. Se o seu sistema operacional não tiver jemalloc, instale-o da seguinte maneira:
 
-    ```bash
-    # ubuntu
-    sudo apt-get install libjemalloc-dev
-    # centos
-    sudo yum instalar jemalloc
-    # mac
-    sudo brew install jemalloc
-    ```
-
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum instalar jemalloc
+   # mac
+   sudo brew install jemalloc
+   ```
 6. Lance o serviço de back-end:
 
    ```bash
@@ -341,14 +339,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. Instale as dependências do front-end:
 
    ```bash
    cd web
    npm install
    ```
-
 8. Lance o serviço de front-end:
 
    ```bash
@@ -358,13 +354,11 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    _O seguinte resultado confirma o lançamento bem-sucedido do sistema:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
-
 9. Pare os serviços de front-end e back-end do RAGFlow após a conclusão do desenvolvimento:
 
-    ```bash
-    pkill -f "ragflow_server.py|task_executor.py"
-    ```
-
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
 
 ## 📚 Documentação
 

README_tzh.md

@@ -1,6 +1,6 @@
 <div align="center">
 <a href="https://demo.ragflow.io/">
-<img src="web/src/assets/logo-with-text.png" width="350" alt="ragflow logo">
+<img src="web/src/assets/logo-with-text.svg" width="350" alt="ragflow logo">
 </a>
 </div>
 
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.20.5">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -43,7 +43,9 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
 
 <div align="center">
 <a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
@@ -83,13 +85,14 @@
 
 ## 🔥 近期更新
 
+- 2025-11-12 支援從 Confluence、AWS S3、Discord、Google Drive 進行資料同步。
+- 2025-10-23 支援 MinerU 和 Docling 作為文件解析方法。
+- 2025-10-15 支援可編排的資料管道。
 - 2025-08-08 支援 OpenAI 最新的 GPT-5 系列模型。
-- 2025-08-04 支援 Kimi K2 和 Grok 4 等模型.
 - 2025-08-01 支援 agentic workflow 和 MCP
 - 2025-05-23 為 Agent 新增 Python/JS 程式碼執行器元件。
 - 2025-05-05 支援跨語言查詢。
 - 2025-03-19 PDF和DOCX中的圖支持用多模態大模型去解析得到描述.
-- 2025-02-28 結合網路搜尋（Tavily），對於任意大模型實現類似 Deep Research 的推理功能.
 - 2024-12-18 升級了 DeepDoc 的文檔佈局分析模型。
 - 2024-08-22 支援用 RAG 技術實現從自然語言到 SQL 語句的轉換。
 
@@ -132,7 +135,7 @@
 ## 🔎 系統架構
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+<img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
 ## 🎬 快速開始
@@ -170,47 +173,52 @@
    > ```bash
    > vm.max_map_count=262144
    > ```
-
+   >
 2. 克隆倉庫：
 
    ```bash
    $ git clone https://github.com/infiniflow/ragflow.git
    ```
-
 3. 進入 **docker** 資料夾，利用事先編譯好的 Docker 映像啟動伺服器：
 
 > [!CAUTION]
 > 所有 Docker 映像檔都是為 x86 平台建置的。目前，我們不提供 ARM64 平台的 Docker 映像檔。
 > 如果您使用的是 ARM64 平台，請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
 
-   > 執行以下指令會自動下載 RAGFlow slim Docker 映像 `v0.20.5-slim`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.20.5-slim` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。例如，你可以透過設定 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.20.5` 來下載 RAGFlow 鏡像的 `v0.20.5` 完整發行版。
+> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.22.0`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.22.0` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
 
-   ```bash
+```bash
    $ cd ragflow/docker
-   # Use CPU for embedding and DeepDoc tasks:
+   
+   # 可選：使用穩定版標籤（查看發佈：https://github.com/infiniflow/ragflow/releases），例：git checkout v0.22.0
+
+   # Use CPU for DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
 
-   # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
-   ```
+   # To use GPU to accelerate DeepDoc tasks:
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
+```
 
-   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-   | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.20.5           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.20.5-slim      | &approx;2       | ❌                    | Stable release           |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+> 注意：在 `v0.22.0` 之前的版本，我們會同時提供包含 embedding 模型的映像和不含 embedding 模型的 slim 映像。具體如下：
 
-   > [!TIP]
-   > 如果你遇到 Docker 映像檔拉不下來的問題，可以在 **docker/.env** 檔案內根據變數 `RAGFLOW_IMAGE` 的註解提示選擇華為雲或阿里雲的對應映像。
-   >
-   > - 華為雲鏡像名：`swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow`
-   > - 阿里雲鏡像名：`registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow`
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
+| ----------------- | --------------- | --------------------- | ------------------------ |
+| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+
+> 從 `v0.22.0` 開始，我們只發佈 slim 版本，並且不再在映像標籤後附加 **-slim** 後綴。
+
+> [!TIP]
+> 如果你遇到 Docker 映像檔拉不下來的問題，可以在 **docker/.env** 檔案內根據變數 `RAGFLOW_IMAGE` 的註解提示選擇華為雲或阿里雲的對應映像。
+>
+> - 華為雲鏡像名：`swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow`
+> - 阿里雲鏡像名：`registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow`
 
 4. 伺服器啟動成功後再次確認伺服器狀態：
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _出現以下介面提示說明伺服器啟動成功：_
@@ -226,12 +234,15 @@
    ```
 
    > 如果您跳過這一步驟系統確認步驟就登入 RAGFlow，你的瀏覽器有可能會提示 `network anormal` 或 `網路異常`，因為 RAGFlow 可能並未完全啟動成功。
-
+   >
 5. 在你的瀏覽器中輸入你的伺服器對應的 IP 位址並登入 RAGFlow。
+
    > 上面這個範例中，您只需輸入 http://IP_OF_YOUR_MACHINE 即可：未改動過設定則無需輸入連接埠（預設的 HTTP 服務連接埠 80）。
+   >
 6. 在 [service_conf.yaml.template](./docker/service_conf.yaml.template) 檔案的 `user_default_llm` 欄位設定 LLM factory，並在 `API_KEY` 欄填入和你選擇的大模型相對應的 API key。
 
    > 詳見 [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup)。
+   >
 
    _好戲開始，接著奏樂接著舞！ _
 
@@ -249,7 +260,7 @@
 
 > [./docker/README](./docker/README.md) 解釋了 [service_conf.yaml.template](./docker/service_conf.yaml.template) 用到的環境變數設定和服務配置。
 
-如需更新預設的 HTTP 服務連接埠(80), 可以在[docker-compose.yml](./docker/docker-compose.yml) 檔案中將配置`80:80` 改為`<YOUR_SERVING_PORT>:80` 。
+如需更新預設的 HTTP 服務連接埠(80), 可以在[docker-compose.yml](./docker/docker-compose.yml) 檔案中將配置 `80:80` 改為 `<YOUR_SERVING_PORT>:80` 。
 
 > 所有系統配置都需要透過系統重新啟動生效：
 >
@@ -266,10 +277,9 @@ RAGFlow 預設使用 Elasticsearch 儲存文字和向量資料. 如果要切換
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+
    Note: `-v` 將會刪除 docker 容器的 volumes，已有的資料會被清空。
-
 2. 設定 **docker/.env** 目錄中的 `DOC_ENGINE` 為 `infinity`.
-
 3. 啟動容器:
 
    ```bash
@@ -279,24 +289,14 @@ RAGFlow 預設使用 Elasticsearch 儲存文字和向量資料. 如果要切換
 > [!WARNING]
 > Infinity 目前官方並未正式支援在 Linux/arm64 架構下的機器上運行.
 
-## 🔧 原始碼編譯 Docker 映像（不含 embedding 模型）
+## 🔧 原始碼編譯 Docker 映像
 
 本 Docker 映像大小約 2 GB 左右並且依賴外部的大模型和 embedding 服務。
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 原始碼編譯 Docker 映像（包含 embedding 模型）
-
-本 Docker 大小約 9 GB 左右。由於已包含 embedding 模型，所以只需依賴外部的大模型服務即可。
-
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 以原始碼啟動服務
@@ -307,17 +307,15 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    pipx install uv pre-commit
    export UV_INDEX=https://mirrors.aliyun.com/pypi/simple
    ```
-
 2. 下載原始碼並安裝 Python 依賴：
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```
-
 3. 透過 Docker Compose 啟動依賴的服務（MinIO, Elasticsearch, Redis, and MySQL）：
 
    ```bash
@@ -329,13 +327,11 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. 如果無法存取 HuggingFace，可以把環境變數 `HF_ENDPOINT` 設為對應的鏡像網站：
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. 如果你的操作系统没有 jemalloc，请按照如下方式安装：
 
    ```bash
@@ -346,7 +342,6 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    # mac
    sudo brew install jemalloc
    ```
-
 6. 啟動後端服務：
 
    ```bash
@@ -354,14 +349,12 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. 安裝前端依賴：
 
    ```bash
    cd web
    npm install
    ```
-
 8. 啟動前端服務：
 
    ```bash
@@ -371,15 +364,16 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    以下界面說明系統已成功啟動：_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
+
    ```
 
+   ```
 9. 開發完成後停止 RAGFlow 前端和後端服務：
 
    ```bash
    pkill -f "ragflow_server.py|task_executor.py"
    ```
 
-
 ## 📚 技術文檔
 
 - [Quickstart](https://ragflow.io/docs/dev/)

README_zh.md

@@ -1,6 +1,6 @@
 <div align="center">
 <a href="https://demo.ragflow.io/">
-<img src="web/src/assets/logo-with-text.png" width="350" alt="ragflow logo">
+<img src="web/src/assets/logo-with-text.svg" width="350" alt="ragflow logo">
 </a>
 </div>
 
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.20.5">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -43,7 +43,9 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
 
 <div align="center">
 <a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
@@ -83,13 +85,14 @@
 
 ## 🔥 近期更新
 
-- 2025-08-08 支持 OpenAI 最新的 GPT-5 系列模型.
-- 2025-08-04 新增对 Kimi K2 和 Grok 4 等模型的支持.
+- 2025-11-12 支持从 Confluence、AWS S3、Discord、Google Drive 进行数据同步。
+- 2025-10-23 支持 MinerU 和 Docling 作为文档解析方法。
+- 2025-10-15 支持可编排的数据管道。
+- 2025-08-08 支持 OpenAI 最新的 GPT-5 系列模型。
 - 2025-08-01 支持 agentic workflow 和 MCP。
 - 2025-05-23 Agent 新增 Python/JS 代码执行器组件。
 - 2025-05-05 支持跨语言查询。
 - 2025-03-19 PDF 和 DOCX 中的图支持用多模态大模型去解析得到描述.
-- 2025-02-28 结合互联网搜索（Tavily），对于任意大模型实现类似 Deep Research 的推理功能.
 - 2024-12-18 升级了 DeepDoc 的文档布局分析模型。
 - 2024-08-22 支持用 RAG 技术实现从自然语言到 SQL 语句的转换。
 
@@ -132,7 +135,7 @@
 ## 🔎 系统架构
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+<img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
 ## 🎬 快速开始
@@ -183,23 +186,29 @@
 > 请注意，目前官方提供的所有 Docker 镜像均基于 x86 架构构建，并不提供基于 ARM64 的 Docker 镜像。
 > 如果你的操作系统是 ARM64 架构，请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
 
-   > 运行以下命令会自动下载 RAGFlow slim Docker 镜像 `v0.20.5-slim`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.20.5-slim` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。比如，你可以通过设置 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.20.5` 来下载 RAGFlow 镜像的 `v0.20.5` 完整发行版。
+   > 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.22.0`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.22.0` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
 
    ```bash
    $ cd ragflow/docker
-   # Use CPU for embedding and DeepDoc tasks:
+   
+   # 可选：使用稳定版本标签（查看发布：https://github.com/infiniflow/ragflow/releases），例如：git checkout v0.22.0
+
+   # Use CPU for DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
 
-   # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
+   # To use GPU to accelerate DeepDoc tasks:
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
    ```
+   
+   > 注意：在 `v0.22.0` 之前的版本，我们会同时提供包含 embedding 模型的镜像和不含 embedding 模型的 slim 镜像。具体如下：
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.20.5           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.20.5-slim      | &approx;2       | ❌                    | Stable release           |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+   | v0.21.1           | &approx;9       | ✔️                    | Stable release           |
+   | v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+
+   > 从 `v0.22.0` 开始，我们只发布 slim 版本，并且不再在镜像标签后附加 **-slim** 后缀。
 
    > [!TIP]
    > 如果你遇到 Docker 镜像拉不下来的问题，可以在 **docker/.env** 文件内根据变量 `RAGFLOW_IMAGE` 的注释提示选择华为云或者阿里云的相应镜像。
@@ -210,7 +219,7 @@
 4. 服务器启动成功后再次确认服务器状态：
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _出现以下界面提示说明服务器启动成功：_
@@ -279,24 +288,14 @@ RAGFlow 默认使用 Elasticsearch 存储文本和向量数据. 如果要切换
 > [!WARNING]
 > Infinity 目前官方并未正式支持在 Linux/arm64 架构下的机器上运行.
 
-## 🔧 源码编译 Docker 镜像（不含 embedding 模型）
+## 🔧 源码编译 Docker 镜像
 
 本 Docker 镜像大小约 2 GB 左右并且依赖外部的大模型和 embedding 服务。
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 源码编译 Docker 镜像（包含 embedding 模型）
-
-本 Docker 大小约 9 GB 左右。由于已包含 embedding 模型，所以只需依赖外部的大模型服务即可。
-
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 以源代码启动服务
@@ -313,7 +312,7 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

admin/build_cli_release.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -e
+
+echo "🚀 Start building..."
+echo "================================"
+
+PROJECT_NAME="ragflow-cli"
+
+RELEASE_DIR="release"
+BUILD_DIR="dist"
+SOURCE_DIR="src"
+PACKAGE_DIR="ragflow_cli"
+
+echo "🧹 Clean old build folder..."
+rm -rf release/
+
+echo "📁 Prepare source code..."
+mkdir release/$PROJECT_NAME/$SOURCE_DIR -p
+cp pyproject.toml release/$PROJECT_NAME/pyproject.toml
+cp README.md release/$PROJECT_NAME/README.md
+
+mkdir release/$PROJECT_NAME/$SOURCE_DIR/$PACKAGE_DIR -p
+cp admin_client.py release/$PROJECT_NAME/$SOURCE_DIR/$PACKAGE_DIR/admin_client.py
+
+if [ -d "release/$PROJECT_NAME/$SOURCE_DIR" ]; then
+    echo "✅ source dir: release/$PROJECT_NAME/$SOURCE_DIR"
+else
+    echo "❌ source dir not exist: release/$PROJECT_NAME/$SOURCE_DIR"
+    exit 1
+fi
+
+echo "🔨 Make build file..."
+cd release/$PROJECT_NAME
+export PYTHONPATH=$(pwd)
+python -m build
+
+echo "✅ check build result..."
+if [ -d "$BUILD_DIR" ]; then
+    echo "📦 Package generated:"
+    ls -la $BUILD_DIR/
+else
+    echo "❌ Build Failed: $BUILD_DIR not exist."
+    exit 1
+fi
+
+echo "🎉 Build finished successfully!"

admin/client/README.md

@@ -0,0 +1,136 @@
+# RAGFlow Admin Service & CLI
+
+### Introduction
+
+Admin Service is a dedicated management component designed to monitor, maintain, and administrate the RAGFlow system. It provides comprehensive tools for ensuring system stability, performing operational tasks, and managing users and permissions efficiently.
+
+The service offers real-time monitoring of critical components, including the RAGFlow server, Task Executor processes, and dependent services such as MySQL, Elasticsearch, Redis, and MinIO. It automatically checks their health status, resource usage, and uptime, and performs restarts in case of failures to minimize downtime.
+
+For user and system management, it supports listing, creating, modifying, and deleting users and their associated resources like knowledge bases and Agents.
+
+Built with scalability and reliability in mind, the Admin Service ensures smooth system operation and simplifies maintenance workflows.
+
+It consists of a server-side Service and a command-line client (CLI), both implemented in Python. User commands are parsed using the Lark parsing toolkit.
+
+- **Admin Service**: A backend service that interfaces with the RAGFlow system to execute administrative operations and monitor its status.
+- **Admin CLI**: A command-line interface that allows users to connect to the Admin Service and issue commands for system management.
+
+
+
+### Starting the Admin Service
+
+#### Launching from source code
+
+1. Before start Admin Service, please make sure RAGFlow system is already started.
+
+2. Launch from source code:
+
+   ```bash
+   python admin/server/admin_server.py
+   ```
+   The service will start and listen for incoming connections from the CLI on the configured port. 
+
+#### Using docker image
+
+1. Before startup, please configure the `docker_compose.yml`  file to enable admin server:
+
+   ```bash
+   command:
+     - --enable-adminserver
+   ```
+
+2. Start the containers, the service will start and listen for incoming connections from the CLI on the configured port.
+
+
+
+### Using the Admin CLI
+
+1.  Ensure the Admin Service is running.
+2.  Install ragflow-cli.
+    ```bash
+    pip install ragflow-cli==0.22.0
+    ```
+3.  Launch the CLI client:
+    ```bash
+    ragflow-cli -h 127.0.0.1 -p 9381
+    ```
+    You will be prompted to enter the superuser's password to log in.
+    The default password is admin.
+
+    **Parameters:**
+    
+    - -h: RAGFlow admin server host address
+    
+    - -p: RAGFlow admin server port
+
+
+
+## Supported Commands
+
+Commands are case-insensitive and must be terminated with a semicolon (`;`).
+
+### Service Management Commands
+
+-   `LIST SERVICES;`
+    -   Lists all available services within the RAGFlow system.
+-   `SHOW SERVICE <id>;`
+    -   Shows detailed status information for the service identified by `<id>`.
+
+
+### User Management Commands
+
+-   `LIST USERS;`
+    -   Lists all users known to the system.
+-   `SHOW USER '<username>';`
+    -   Shows details and permissions for the specified user. The username must be enclosed in single or double quotes.
+
+- `CREATE USER <username> <password>;`
+  - Create user by username and password. The username and password must be enclosed in single or double quotes.
+
+-   `DROP USER '<username>';`
+    -   Removes the specified user from the system. Use with caution.
+-   `ALTER USER PASSWORD '<username>' '<new_password>';`
+    -   Changes the password for the specified user.
+-   `ALTER USER ACTIVE <username> <on/off>;`
+    -   Changes the user to active or inactive.
+
+
+### Data and Agent Commands
+
+-   `LIST DATASETS OF '<username>';`
+    -   Lists the datasets associated with the specified user.
+-   `LIST AGENTS OF '<username>';`
+    -   Lists the agents associated with the specified user.
+
+### Meta-Commands
+
+Meta-commands are prefixed with a backslash (`\`).
+
+-   `\?` or `\help`
+    -   Shows help information for the available commands.
+-   `\q` or `\quit`
+    -   Exits the CLI application.
+
+## Examples
+
+```commandline
+admin> list users;
++-------------------------------+------------------------+-----------+-------------+
+| create_date                   | email                  | is_active | nickname    |
++-------------------------------+------------------------+-----------+-------------+
+| Fri, 22 Nov 2024 16:03:41 GMT | jeffery@infiniflow.org | 1         | Jeffery     |
+| Fri, 22 Nov 2024 16:10:55 GMT | aya@infiniflow.org     | 1         | Waterdancer |
++-------------------------------+------------------------+-----------+-------------+
+
+admin> list services;
++-------------------------------------------------------------------------------------------+-----------+----+---------------+-------+----------------+
+| extra                                                                                     | host      | id | name          | port  | service_type   |
++-------------------------------------------------------------------------------------------+-----------+----+---------------+-------+----------------+
+| {}                                                                                        | 0.0.0.0   | 0  | ragflow_0     | 9380  | ragflow_server |
+| {'meta_type': 'mysql', 'password': 'infini_rag_flow', 'username': 'root'}                 | localhost | 1  | mysql         | 5455  | meta_data      |
+| {'password': 'infini_rag_flow', 'store_type': 'minio', 'user': 'rag_flow'}                | localhost | 2  | minio         | 9000  | file_store     |
+| {'password': 'infini_rag_flow', 'retrieval_type': 'elasticsearch', 'username': 'elastic'} | localhost | 3  | elasticsearch | 1200  | retrieval      |
+| {'db_name': 'default_db', 'retrieval_type': 'infinity'}                                   | localhost | 4  | infinity      | 23817 | retrieval      |
+| {'database': 1, 'mq_type': 'redis', 'password': 'infini_rag_flow'}                        | localhost | 5  | redis         | 6379  | message_queue  |
++-------------------------------------------------------------------------------------------+-----------+----+---------------+-------+----------------+
+```

admin/client/admin_client.py

@@ -0,0 +1,975 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import argparse
+import base64
+from cmd import Cmd
+
+from Cryptodome.PublicKey import RSA
+from Cryptodome.Cipher import PKCS1_v1_5 as Cipher_pkcs1_v1_5
+from typing import Dict, List, Any
+from lark import Lark, Transformer, Tree
+import requests
+import getpass
+
+GRAMMAR = r"""
+start: command
+
+command: sql_command | meta_command
+
+sql_command: list_services
+           | show_service
+           | startup_service
+           | shutdown_service
+           | restart_service
+           | list_users
+           | show_user
+           | drop_user
+           | alter_user
+           | create_user
+           | activate_user
+           | list_datasets
+           | list_agents
+           | create_role
+           | drop_role
+           | alter_role
+           | list_roles
+           | show_role
+           | grant_permission
+           | revoke_permission
+           | alter_user_role
+           | show_user_permission
+           | show_version
+
+// meta command definition
+meta_command: "\\" meta_command_name [meta_args]
+
+meta_command_name: /[a-zA-Z?]+/
+meta_args: (meta_arg)+
+
+meta_arg: /[^\\s"']+/ | quoted_string
+
+// command definition
+
+LIST: "LIST"i
+SERVICES: "SERVICES"i
+SHOW: "SHOW"i
+CREATE: "CREATE"i
+SERVICE: "SERVICE"i
+SHUTDOWN: "SHUTDOWN"i
+STARTUP: "STARTUP"i
+RESTART: "RESTART"i
+USERS: "USERS"i
+DROP: "DROP"i
+USER: "USER"i
+ALTER: "ALTER"i
+ACTIVE: "ACTIVE"i
+PASSWORD: "PASSWORD"i
+DATASETS: "DATASETS"i
+OF: "OF"i
+AGENTS: "AGENTS"i
+ROLE: "ROLE"i
+ROLES: "ROLES"i
+DESCRIPTION: "DESCRIPTION"i
+GRANT: "GRANT"i
+REVOKE: "REVOKE"i
+ALL: "ALL"i
+PERMISSION: "PERMISSION"i
+TO: "TO"i
+FROM: "FROM"i
+FOR: "FOR"i
+RESOURCES: "RESOURCES"i
+ON: "ON"i
+SET: "SET"i
+VERSION: "VERSION"i
+
+list_services: LIST SERVICES ";"
+show_service: SHOW SERVICE NUMBER ";"
+startup_service: STARTUP SERVICE NUMBER ";"
+shutdown_service: SHUTDOWN SERVICE NUMBER ";"
+restart_service: RESTART SERVICE NUMBER ";"
+
+list_users: LIST USERS ";"
+drop_user: DROP USER quoted_string ";"
+alter_user: ALTER USER PASSWORD quoted_string quoted_string ";"
+show_user: SHOW USER quoted_string ";"
+create_user: CREATE USER quoted_string quoted_string ";"
+activate_user: ALTER USER ACTIVE quoted_string status ";"
+
+list_datasets: LIST DATASETS OF quoted_string ";"
+list_agents: LIST AGENTS OF quoted_string ";"
+
+create_role: CREATE ROLE identifier [DESCRIPTION quoted_string] ";"
+drop_role: DROP ROLE identifier ";"
+alter_role: ALTER ROLE identifier SET DESCRIPTION quoted_string ";"
+list_roles: LIST ROLES ";"
+show_role: SHOW ROLE identifier ";"
+
+grant_permission: GRANT action_list ON identifier TO ROLE identifier ";"
+revoke_permission: REVOKE action_list ON identifier FROM ROLE identifier ";"
+alter_user_role: ALTER USER quoted_string SET ROLE identifier ";"
+show_user_permission: SHOW USER PERMISSION quoted_string ";"
+
+show_version: SHOW VERSION ";"
+
+action_list: identifier ("," identifier)*
+
+identifier: WORD
+quoted_string: QUOTED_STRING
+status: WORD
+
+QUOTED_STRING: /'[^']+'/ | /"[^"]+"/
+WORD: /[a-zA-Z0-9_\-\.]+/
+NUMBER: /[0-9]+/
+
+%import common.WS
+%ignore WS
+"""
+
+
+class AdminTransformer(Transformer):
+
+    def start(self, items):
+        return items[0]
+
+    def command(self, items):
+        return items[0]
+
+    def list_services(self, items):
+        result = {'type': 'list_services'}
+        return result
+
+    def show_service(self, items):
+        service_id = int(items[2])
+        return {"type": "show_service", "number": service_id}
+
+    def startup_service(self, items):
+        service_id = int(items[2])
+        return {"type": "startup_service", "number": service_id}
+
+    def shutdown_service(self, items):
+        service_id = int(items[2])
+        return {"type": "shutdown_service", "number": service_id}
+
+    def restart_service(self, items):
+        service_id = int(items[2])
+        return {"type": "restart_service", "number": service_id}
+
+    def list_users(self, items):
+        return {"type": "list_users"}
+
+    def show_user(self, items):
+        user_name = items[2]
+        return {"type": "show_user", "user_name": user_name}
+
+    def drop_user(self, items):
+        user_name = items[2]
+        return {"type": "drop_user", "user_name": user_name}
+
+    def alter_user(self, items):
+        user_name = items[3]
+        new_password = items[4]
+        return {"type": "alter_user", "user_name": user_name, "password": new_password}
+
+    def create_user(self, items):
+        user_name = items[2]
+        password = items[3]
+        return {"type": "create_user", "user_name": user_name, "password": password, "role": "user"}
+
+    def activate_user(self, items):
+        user_name = items[3]
+        activate_status = items[4]
+        return {"type": "activate_user", "activate_status": activate_status, "user_name": user_name}
+
+    def list_datasets(self, items):
+        user_name = items[3]
+        return {"type": "list_datasets", "user_name": user_name}
+
+    def list_agents(self, items):
+        user_name = items[3]
+        return {"type": "list_agents", "user_name": user_name}
+
+    def create_role(self, items):
+        role_name = items[2]
+        if len(items) > 4:
+            description = items[4]
+            return {"type": "create_role", "role_name": role_name, "description": description}
+        else:
+            return {"type": "create_role", "role_name": role_name}
+
+    def drop_role(self, items):
+        role_name = items[2]
+        return {"type": "drop_role", "role_name": role_name}
+
+    def alter_role(self, items):
+        role_name = items[2]
+        description = items[5]
+        return {"type": "alter_role", "role_name": role_name, "description": description}
+
+    def list_roles(self, items):
+        return {"type": "list_roles"}
+
+    def show_role(self, items):
+        role_name = items[2]
+        return {"type": "show_role", "role_name": role_name}
+
+    def grant_permission(self, items):
+        action_list = items[1]
+        resource = items[3]
+        role_name = items[6]
+        return {"type": "grant_permission", "role_name": role_name, "resource": resource, "actions": action_list}
+
+    def revoke_permission(self, items):
+        action_list = items[1]
+        resource = items[3]
+        role_name = items[6]
+        return {
+            "type": "revoke_permission",
+            "role_name": role_name,
+            "resource": resource, "actions": action_list
+        }
+
+    def alter_user_role(self, items):
+        user_name = items[2]
+        role_name = items[5]
+        return {"type": "alter_user_role", "user_name": user_name, "role_name": role_name}
+
+    def show_user_permission(self, items):
+        user_name = items[3]
+        return {"type": "show_user_permission", "user_name": user_name}
+
+    def show_version(self, items):
+        return {"type": "show_version"}
+
+    def action_list(self, items):
+        return items
+
+    def meta_command(self, items):
+        command_name = str(items[0]).lower()
+        args = items[1:] if len(items) > 1 else []
+
+        # handle quoted parameter
+        parsed_args = []
+        for arg in args:
+            if hasattr(arg, 'value'):
+                parsed_args.append(arg.value)
+            else:
+                parsed_args.append(str(arg))
+
+        return {'type': 'meta', 'command': command_name, 'args': parsed_args}
+
+    def meta_command_name(self, items):
+        return items[0]
+
+    def meta_args(self, items):
+        return items
+
+
+def encrypt(input_string):
+    pub = '-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArq9XTUSeYr2+N1h3Afl/z8Dse/2yD0ZGrKwx+EEEcdsBLca9Ynmx3nIB5obmLlSfmskLpBo0UACBmB5rEjBp2Q2f3AG3Hjd4B+gNCG6BDaawuDlgANIhGnaTLrIqWrrcm4EMzJOnAOI1fgzJRsOOUEfaS318Eq9OVO3apEyCCt0lOQK6PuksduOjVxtltDav+guVAA068NrPYmRNabVKRNLJpL8w4D44sfth5RvZ3q9t+6RTArpEtc5sh5ChzvqPOzKGMXW83C95TxmXqpbK6olN4RevSfVjEAgCydH6HN6OhtOQEcnrU97r9H0iZOWwbw3pVrZiUkuRD1R56Wzs2wIDAQAB\n-----END PUBLIC KEY-----'
+    pub_key = RSA.importKey(pub)
+    cipher = Cipher_pkcs1_v1_5.new(pub_key)
+    cipher_text = cipher.encrypt(base64.b64encode(input_string.encode('utf-8')))
+    return base64.b64encode(cipher_text).decode("utf-8")
+
+
+def encode_to_base64(input_string):
+    base64_encoded = base64.b64encode(input_string.encode('utf-8'))
+    return base64_encoded.decode('utf-8')
+
+
+class AdminCLI(Cmd):
+    def __init__(self):
+        super().__init__()
+        self.parser = Lark(GRAMMAR, start='start', parser='lalr', transformer=AdminTransformer())
+        self.command_history = []
+        self.is_interactive = False
+        self.admin_account = "admin@ragflow.io"
+        self.admin_password: str = "admin"
+        self.session = requests.Session()
+        self.access_token: str = ""
+        self.host: str = ""
+        self.port: int = 0
+
+    intro = r"""Type "\h" for help."""
+    prompt = "admin> "
+
+    def onecmd(self, command: str) -> bool:
+        try:
+            result = self.parse_command(command)
+
+            if isinstance(result, dict):
+                if 'type' in result and result.get('type') == 'empty':
+                    return False
+
+            self.execute_command(result)
+
+            if isinstance(result, Tree):
+                return False
+
+            if result.get('type') == 'meta' and result.get('command') in ['q', 'quit', 'exit']:
+                return True
+
+        except KeyboardInterrupt:
+            print("\nUse '\\q' to quit")
+        except EOFError:
+            print("\nGoodbye!")
+            return True
+        return False
+
+    def emptyline(self) -> bool:
+        return False
+
+    def default(self, line: str) -> bool:
+        return self.onecmd(line)
+
+    def parse_command(self, command_str: str) -> dict[str, str]:
+        if not command_str.strip():
+            return {'type': 'empty'}
+
+        self.command_history.append(command_str)
+
+        try:
+            result = self.parser.parse(command_str)
+            return result
+        except Exception as e:
+            return {'type': 'error', 'message': f'Parse error: {str(e)}'}
+
+    def verify_admin(self, arguments: dict, single_command: bool):
+        self.host = arguments['host']
+        self.port = arguments['port']
+        print(f"Attempt to access ip: {self.host}, port: {self.port}")
+        url = f"http://{self.host}:{self.port}/api/v1/admin/login"
+
+        attempt_count = 3
+        if single_command:
+            attempt_count = 1
+
+        try_count = 0
+        while True:
+            try_count += 1
+            if try_count > attempt_count:
+                return False
+
+            if single_command:
+                admin_passwd = arguments['password']
+            else:
+                admin_passwd = getpass.getpass(f"password for {self.admin_account}: ").strip()
+            try:
+                self.admin_password = encrypt(admin_passwd)
+                response = self.session.post(url, json={'email': self.admin_account, 'password': self.admin_password})
+                if response.status_code == 200:
+                    res_json = response.json()
+                    error_code = res_json.get('code', -1)
+                    if error_code == 0:
+                        self.session.headers.update({
+                            'Content-Type': 'application/json',
+                            'Authorization': response.headers['Authorization'],
+                            'User-Agent': 'RAGFlow-CLI/0.22.0'
+                        })
+                        print("Authentication successful.")
+                        return True
+                    else:
+                        error_message = res_json.get('message', 'Unknown error')
+                        print(f"Authentication failed: {error_message}, try again")
+                        continue
+                else:
+                    print(f"Bad response，status: {response.status_code}, password is wrong")
+            except Exception as e:
+                print(str(e))
+                print(f"Can't access {self.host}, port: {self.port}")
+
+    def _format_service_detail_table(self, data):
+        if not any([isinstance(v, list) for v in data.values()]):
+            # normal table
+            return data
+        # handle task_executor heartbeats map, for example {'name': [{'done': 2, 'now': timestamp1}, {'done': 3, 'now': timestamp2}]
+        task_executor_list = []
+        for k, v in data.items():
+            # display latest status
+            heartbeats = sorted(v, key=lambda x: x["now"], reverse=True)
+            task_executor_list.append({
+                "task_executor_name": k,
+                **heartbeats[0],
+            })
+        return task_executor_list
+
+    def _print_table_simple(self, data):
+        if not data:
+            print("No data to print")
+            return
+        if isinstance(data, dict):
+            # handle single row data
+            data = [data]
+
+        columns = list(data[0].keys())
+        col_widths = {}
+
+        def get_string_width(text):
+            half_width_chars = (
+                " !\"#$%&'()*+,-./0123456789:;<=>?@"
+                "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`"
+                "abcdefghijklmnopqrstuvwxyz{|}~"
+                "\t\n\r"
+            )
+            width = 0
+            for char in text:
+                if char in half_width_chars:
+                    width += 1
+                else:
+                    width += 2
+            return width
+
+        for col in columns:
+            max_width = get_string_width(str(col))
+            for item in data:
+                value_len = get_string_width(str(item.get(col, '')))
+                if value_len > max_width:
+                    max_width = value_len
+            col_widths[col] = max(2, max_width)
+
+        # Generate delimiter
+        separator = "+" + "+".join(["-" * (col_widths[col] + 2) for col in columns]) + "+"
+
+        # Print header
+        print(separator)
+        header = "|" + "|".join([f" {col:<{col_widths[col]}} " for col in columns]) + "|"
+        print(header)
+        print(separator)
+
+        # Print data
+        for item in data:
+            row = "|"
+            for col in columns:
+                value = str(item.get(col, ''))
+                if get_string_width(value) > col_widths[col]:
+                    value = value[:col_widths[col] - 3] + "..."
+                row += f" {value:<{col_widths[col] - (get_string_width(value) - len(value))}} |"
+            print(row)
+
+        print(separator)
+
+    def run_interactive(self):
+
+        self.is_interactive = True
+        print("RAGFlow Admin command line interface - Type '\\?' for help, '\\q' to quit")
+
+        while True:
+            try:
+                command = input("admin> ").strip()
+                if not command:
+                    continue
+
+                print(f"command: {command}")
+                result = self.parse_command(command)
+                self.execute_command(result)
+
+                if isinstance(result, Tree):
+                    continue
+
+                if result.get('type') == 'meta' and result.get('command') in ['q', 'quit', 'exit']:
+                    break
+
+            except KeyboardInterrupt:
+                print("\nUse '\\q' to quit")
+            except EOFError:
+                print("\nGoodbye!")
+                break
+
+    def run_single_command(self, command: str):
+        result = self.parse_command(command)
+        self.execute_command(result)
+
+    def parse_connection_args(self, args: List[str]) -> Dict[str, Any]:
+        parser = argparse.ArgumentParser(description='Admin CLI Client', add_help=False)
+        parser.add_argument('-h', '--host', default='localhost', help='Admin service host')
+        parser.add_argument('-p', '--port', type=int, default=9381, help='Admin service port')
+        parser.add_argument('-w', '--password', default='admin', type=str, help='Superuser password')
+        parser.add_argument('command', nargs='?', help='Single command')
+        try:
+            parsed_args, remaining_args = parser.parse_known_args(args)
+            if remaining_args:
+                command = remaining_args[0]
+                return {
+                    'host': parsed_args.host,
+                    'port': parsed_args.port,
+                    'password': parsed_args.password,
+                    'command': command
+                }
+            else:
+                return {
+                    'host': parsed_args.host,
+                    'port': parsed_args.port,
+                }
+        except SystemExit:
+            return {'error': 'Invalid connection arguments'}
+
+    def execute_command(self, parsed_command: Dict[str, Any]):
+
+        command_dict: dict
+        if isinstance(parsed_command, Tree):
+            command_dict = parsed_command.children[0]
+        else:
+            if parsed_command['type'] == 'error':
+                print(f"Error: {parsed_command['message']}")
+                return
+            else:
+                command_dict = parsed_command
+
+        # print(f"Parsed command: {command_dict}")
+
+        command_type = command_dict['type']
+
+        match command_type:
+            case 'list_services':
+                self._handle_list_services(command_dict)
+            case 'show_service':
+                self._handle_show_service(command_dict)
+            case 'restart_service':
+                self._handle_restart_service(command_dict)
+            case 'shutdown_service':
+                self._handle_shutdown_service(command_dict)
+            case 'startup_service':
+                self._handle_startup_service(command_dict)
+            case 'list_users':
+                self._handle_list_users(command_dict)
+            case 'show_user':
+                self._handle_show_user(command_dict)
+            case 'drop_user':
+                self._handle_drop_user(command_dict)
+            case 'alter_user':
+                self._handle_alter_user(command_dict)
+            case 'create_user':
+                self._handle_create_user(command_dict)
+            case 'activate_user':
+                self._handle_activate_user(command_dict)
+            case 'list_datasets':
+                self._handle_list_datasets(command_dict)
+            case 'list_agents':
+                self._handle_list_agents(command_dict)
+            case 'create_role':
+                self._create_role(command_dict)
+            case 'drop_role':
+                self._drop_role(command_dict)
+            case 'alter_role':
+                self._alter_role(command_dict)
+            case 'list_roles':
+                self._list_roles(command_dict)
+            case 'show_role':
+                self._show_role(command_dict)
+            case 'grant_permission':
+                self._grant_permission(command_dict)
+            case 'revoke_permission':
+                self._revoke_permission(command_dict)
+            case 'alter_user_role':
+                self._alter_user_role(command_dict)
+            case 'show_user_permission':
+                self._show_user_permission(command_dict)
+            case 'show_version':
+                self._show_version(command_dict)
+            case 'meta':
+                self._handle_meta_command(command_dict)
+            case _:
+                print(f"Command '{command_type}' would be executed with API")
+
+    def _handle_list_services(self, command):
+        print("Listing all services")
+
+        url = f'http://{self.host}:{self.port}/api/v1/admin/services'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(f"Fail to get all services, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _handle_show_service(self, command):
+        service_id: int = command['number']
+        print(f"Showing service: {service_id}")
+
+        url = f'http://{self.host}:{self.port}/api/v1/admin/services/{service_id}'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            res_data = res_json['data']
+            if 'status' in res_data and res_data['status'] == 'alive':
+                print(f"Service {res_data['service_name']} is alive, ")
+                if isinstance(res_data['message'], str):
+                    print(res_data['message'])
+                else:
+                    data = self._format_service_detail_table(res_data['message'])
+                    self._print_table_simple(data)
+            else:
+                print(f"Service {res_data['service_name']} is down, {res_data['message']}")
+        else:
+            print(f"Fail to show service, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _handle_restart_service(self, command):
+        service_id: int = command['number']
+        print(f"Restart service {service_id}")
+
+    def _handle_shutdown_service(self, command):
+        service_id: int = command['number']
+        print(f"Shutdown service {service_id}")
+
+    def _handle_startup_service(self, command):
+        service_id: int = command['number']
+        print(f"Startup service {service_id}")
+
+    def _handle_list_users(self, command):
+        print("Listing all users")
+
+        url = f'http://{self.host}:{self.port}/api/v1/admin/users'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(f"Fail to get all users, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _handle_show_user(self, command):
+        username_tree: Tree = command['user_name']
+        user_name: str = username_tree.children[0].strip("'\"")
+        print(f"Showing user: {user_name}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            table_data = res_json['data']
+            table_data.pop('avatar')
+            self._print_table_simple(table_data)
+        else:
+            print(f"Fail to get user {user_name}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _handle_drop_user(self, command):
+        username_tree: Tree = command['user_name']
+        user_name: str = username_tree.children[0].strip("'\"")
+        print(f"Drop user: {user_name}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}'
+        response = self.session.delete(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            print(res_json["message"])
+        else:
+            print(f"Fail to drop user, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _handle_alter_user(self, command):
+        user_name_tree: Tree = command['user_name']
+        user_name: str = user_name_tree.children[0].strip("'\"")
+        password_tree: Tree = command['password']
+        password: str = password_tree.children[0].strip("'\"")
+        print(f"Alter user: {user_name}, password: {password}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/password'
+        response = self.session.put(url, json={'new_password': encrypt(password)})
+        res_json = response.json()
+        if response.status_code == 200:
+            print(res_json["message"])
+        else:
+            print(f"Fail to alter password, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _handle_create_user(self, command):
+        user_name_tree: Tree = command['user_name']
+        user_name: str = user_name_tree.children[0].strip("'\"")
+        password_tree: Tree = command['password']
+        password: str = password_tree.children[0].strip("'\"")
+        role: str = command['role']
+        print(f"Create user: {user_name}, password: {password}, role: {role}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/users'
+        response = self.session.post(
+            url,
+            json={'user_name': user_name, 'password': encrypt(password), 'role': role}
+        )
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(f"Fail to create user {user_name}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _handle_activate_user(self, command):
+        user_name_tree: Tree = command['user_name']
+        user_name: str = user_name_tree.children[0].strip("'\"")
+        activate_tree: Tree = command['activate_status']
+        activate_status: str = activate_tree.children[0].strip("'\"")
+        if activate_status.lower() in ['on', 'off']:
+            print(f"Alter user {user_name} activate status, turn {activate_status.lower()}.")
+            url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/activate'
+            response = self.session.put(url, json={'activate_status': activate_status})
+            res_json = response.json()
+            if response.status_code == 200:
+                print(res_json["message"])
+            else:
+                print(f"Fail to alter activate status, code: {res_json['code']}, message: {res_json['message']}")
+        else:
+            print(f"Unknown activate status: {activate_status}.")
+
+    def _handle_list_datasets(self, command):
+        username_tree: Tree = command['user_name']
+        user_name: str = username_tree.children[0].strip("'\"")
+        print(f"Listing all datasets of user: {user_name}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/datasets'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            table_data = res_json['data']
+            for t in table_data:
+                t.pop('avatar')
+            self._print_table_simple(table_data)
+        else:
+            print(f"Fail to get all datasets of {user_name}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _handle_list_agents(self, command):
+        username_tree: Tree = command['user_name']
+        user_name: str = username_tree.children[0].strip("'\"")
+        print(f"Listing all agents of user: {user_name}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/agents'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            table_data = res_json['data']
+            for t in table_data:
+                t.pop('avatar')
+            self._print_table_simple(table_data)
+        else:
+            print(f"Fail to get all agents of {user_name}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _create_role(self, command):
+        role_name_tree: Tree = command['role_name']
+        role_name: str = role_name_tree.children[0].strip("'\"")
+        desc_str: str = ''
+        if 'description' in command:
+            desc_tree: Tree = command['description']
+            desc_str = desc_tree.children[0].strip("'\"")
+
+        print(f"create role name: {role_name}, description: {desc_str}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/roles'
+        response = self.session.post(
+            url,
+            json={'role_name': role_name, 'description': desc_str}
+        )
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(f"Fail to create role {role_name}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _drop_role(self, command):
+        role_name_tree: Tree = command['role_name']
+        role_name: str = role_name_tree.children[0].strip("'\"")
+        print(f"drop role name: {role_name}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name}'
+        response = self.session.delete(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(f"Fail to drop role {role_name}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _alter_role(self, command):
+        role_name_tree: Tree = command['role_name']
+        role_name: str = role_name_tree.children[0].strip("'\"")
+        desc_tree: Tree = command['description']
+        desc_str: str = desc_tree.children[0].strip("'\"")
+
+        print(f"alter role name: {role_name}, description: {desc_str}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name}'
+        response = self.session.put(
+            url,
+            json={'description': desc_str}
+        )
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(
+                f"Fail to update role {role_name} with description: {desc_str}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _list_roles(self, command):
+        print("Listing all roles")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/roles'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(f"Fail to list roles, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _show_role(self, command):
+        role_name_tree: Tree = command['role_name']
+        role_name: str = role_name_tree.children[0].strip("'\"")
+        print(f"show role: {role_name}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name}/permission'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(f"Fail to list roles, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _grant_permission(self, command):
+        role_name_tree: Tree = command['role_name']
+        role_name_str: str = role_name_tree.children[0].strip("'\"")
+        resource_tree: Tree = command['resource']
+        resource_str: str = resource_tree.children[0].strip("'\"")
+        action_tree_list: list = command['actions']
+        actions: list = []
+        for action_tree in action_tree_list:
+            action_str: str = action_tree.children[0].strip("'\"")
+            actions.append(action_str)
+        print(f"grant role_name: {role_name_str}, resource: {resource_str}, actions: {actions}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name_str}/permission'
+        response = self.session.post(
+            url,
+            json={'actions': actions, 'resource': resource_str}
+        )
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(
+                f"Fail to grant role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _revoke_permission(self, command):
+        role_name_tree: Tree = command['role_name']
+        role_name_str: str = role_name_tree.children[0].strip("'\"")
+        resource_tree: Tree = command['resource']
+        resource_str: str = resource_tree.children[0].strip("'\"")
+        action_tree_list: list = command['actions']
+        actions: list = []
+        for action_tree in action_tree_list:
+            action_str: str = action_tree.children[0].strip("'\"")
+            actions.append(action_str)
+        print(f"revoke role_name: {role_name_str}, resource: {resource_str}, actions: {actions}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name_str}/permission'
+        response = self.session.delete(
+            url,
+            json={'actions': actions, 'resource': resource_str}
+        )
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(
+                f"Fail to revoke role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _alter_user_role(self, command):
+        role_name_tree: Tree = command['role_name']
+        role_name_str: str = role_name_tree.children[0].strip("'\"")
+        user_name_tree: Tree = command['user_name']
+        user_name_str: str = user_name_tree.children[0].strip("'\"")
+        print(f"alter_user_role user_name: {user_name_str}, role_name: {role_name_str}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name_str}/role'
+        response = self.session.put(
+            url,
+            json={'role_name': role_name_str}
+        )
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(
+                f"Fail to alter user: {user_name_str} to role {role_name_str}, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _show_user_permission(self, command):
+        user_name_tree: Tree = command['user_name']
+        user_name_str: str = user_name_tree.children[0].strip("'\"")
+        print(f"show_user_permission user_name: {user_name_str}")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name_str}/permission'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(
+                f"Fail to show user: {user_name_str} permission, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _show_version(self, command):
+        print("show_version")
+        url = f'http://{self.host}:{self.port}/api/v1/admin/version'
+        response = self.session.get(url)
+        res_json = response.json()
+        if response.status_code == 200:
+            self._print_table_simple(res_json['data'])
+        else:
+            print(f"Fail to show version, code: {res_json['code']}, message: {res_json['message']}")
+
+    def _handle_meta_command(self, command):
+        meta_command = command['command']
+        args = command.get('args', [])
+
+        if meta_command in ['?', 'h', 'help']:
+            self.show_help()
+        elif meta_command in ['q', 'quit', 'exit']:
+            print("Goodbye!")
+        else:
+            print(f"Meta command '{meta_command}' with args {args}")
+
+    def show_help(self):
+        """Help info"""
+        help_text = """
+Commands:
+  LIST SERVICES
+  SHOW SERVICE <service>
+  STARTUP SERVICE <service>
+  SHUTDOWN SERVICE <service>
+  RESTART SERVICE <service>
+  LIST USERS
+  SHOW USER <user>
+  DROP USER <user>
+  CREATE USER <user> <password>
+  ALTER USER PASSWORD <user> <new_password>
+  ALTER USER ACTIVE <user> <on/off>
+  LIST DATASETS OF <user>
+  LIST AGENTS OF <user>
+
+Meta Commands:
+  \\?, \\h, \\help     Show this help
+  \\q, \\quit, \\exit   Quit the CLI
+        """
+        print(help_text)
+
+
+def main():
+    import sys
+
+    cli = AdminCLI()
+
+    args = cli.parse_connection_args(sys.argv)
+    if 'error' in args:
+        print(f"Error: {args['error']}")
+        return
+
+    if 'command' in args:
+        if 'password' not in args:
+            print("Error: password is missing")
+            return
+        if cli.verify_admin(args, single_command=True):
+            command: str = args['command']
+            print(f"Run single command: {command}")
+            cli.run_single_command(command)
+    else:
+        if cli.verify_admin(args, single_command=False):
+            print(r"""
+                ____  ___   ______________                 ___       __          _     
+               / __ \/   | / ____/ ____/ /___ _      __   /   | ____/ /___ ___  (_)___ 
+              / /_/ / /| |/ / __/ /_  / / __ \ | /| / /  / /| |/ __  / __ `__ \/ / __ \
+             / _, _/ ___ / /_/ / __/ / / /_/ / |/ |/ /  / ___ / /_/ / / / / / / / / / /
+            /_/ |_/_/  |_\____/_/   /_/\____/|__/|__/  /_/  |_\__,_/_/ /_/ /_/_/_/ /_/ 
+            """)
+            cli.cmdloop()
+
+
+if __name__ == '__main__':
+    main()

admin/client/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "ragflow-cli"
+version = "0.22.0"
+description = "Admin Service's client of [RAGFlow](https://github.com/infiniflow/ragflow). The Admin Service provides user management and system monitoring. "
+authors = [{ name = "Lynn", email = "lynn_inf@hotmail.com" }]
+license = { text = "Apache License, Version 2.0" }
+readme = "README.md"
+requires-python = ">=3.10,<3.13"
+dependencies = [
+    "requests>=2.30.0,<3.0.0",
+    "beartype>=0.18.5,<0.19.0",
+    "pycryptodomex>=3.10.0",
+    "lark>=1.1.0",
+]
+
+[dependency-groups]
+test = [
+    "pytest>=8.3.5",
+    "requests>=2.32.3",
+    "requests-toolbelt>=1.0.0",
+]
+
+[project.scripts]
+ragflow-cli = "admin_client:main"

admin/server/admin_server.py

@@ -0,0 +1,79 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import os
+import signal
+import logging
+import time
+import threading
+import traceback
+from werkzeug.serving import run_simple
+from flask import Flask
+from routes import admin_bp
+from common.log_utils import init_root_logger
+from common.constants import SERVICE_CONF
+from common.config_utils import show_configs
+from common import settings
+from config import load_configurations, SERVICE_CONFIGS
+from auth import init_default_admin, setup_auth
+from flask_session import Session
+from flask_login import LoginManager
+from common.versions import get_ragflow_version
+
+stop_event = threading.Event()
+
+if __name__ == '__main__':
+    init_root_logger("admin_service")
+    logging.info(r"""
+        ____  ___   ______________                 ___       __          _     
+       / __ \/   | / ____/ ____/ /___ _      __   /   | ____/ /___ ___  (_)___ 
+      / /_/ / /| |/ / __/ /_  / / __ \ | /| / /  / /| |/ __  / __ `__ \/ / __ \
+     / _, _/ ___ / /_/ / __/ / / /_/ / |/ |/ /  / ___ / /_/ / / / / / / / / / /
+    /_/ |_/_/  |_\____/_/   /_/\____/|__/|__/  /_/  |_\__,_/_/ /_/ /_/_/_/ /_/ 
+    """)
+
+    app = Flask(__name__)
+    app.register_blueprint(admin_bp)
+    app.config["SESSION_PERMANENT"] = False
+    app.config["SESSION_TYPE"] = "filesystem"
+    app.config["MAX_CONTENT_LENGTH"] = int(
+        os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024 * 1024)
+    )
+    Session(app)
+    logging.info(f'RAGFlow version: {get_ragflow_version()}')
+    show_configs()
+    login_manager = LoginManager()
+    login_manager.init_app(app)
+    settings.init_settings()
+    setup_auth(login_manager)
+    init_default_admin()
+    SERVICE_CONFIGS.configs = load_configurations(SERVICE_CONF)
+
+    try:
+        logging.info("RAGFlow Admin service start...")
+        run_simple(
+            hostname="0.0.0.0",
+            port=9381,
+            application=app,
+            threaded=True,
+            use_reloader=False,
+            use_debugger=True,
+        )
+    except Exception:
+        traceback.print_exc()
+        stop_event.set()
+        time.sleep(1)
+        os.kill(os.getpid(), signal.SIGKILL)

admin/server/auth.py

@@ -0,0 +1,187 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+import logging
+import uuid
+from functools import wraps
+from datetime import datetime
+from flask import request, jsonify
+from flask_login import current_user, login_user
+from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
+
+from api.common.exceptions import AdminException, UserNotFoundError
+from api.common.base64 import encode_to_base64
+from api.db.services import UserService
+from common.constants import ActiveEnum, StatusEnum
+from api.utils.crypt import decrypt
+from common.misc_utils import get_uuid
+from common.time_utils import current_timestamp, datetime_format, get_format_time
+from common.connection_utils import construct_response
+from common import settings
+
+
+def setup_auth(login_manager):
+    @login_manager.request_loader
+    def load_user(web_request):
+        jwt = Serializer(secret_key=settings.SECRET_KEY)
+        authorization = web_request.headers.get("Authorization")
+        if authorization:
+            try:
+                access_token = str(jwt.loads(authorization))
+
+                if not access_token or not access_token.strip():
+                    logging.warning("Authentication attempt with empty access token")
+                    return None
+
+                # Access tokens should be UUIDs (32 hex characters)
+                if len(access_token.strip()) < 32:
+                    logging.warning(f"Authentication attempt with invalid token format: {len(access_token)} chars")
+                    return None
+
+                user = UserService.query(
+                    access_token=access_token, status=StatusEnum.VALID.value
+                )
+                if user:
+                    if not user[0].access_token or not user[0].access_token.strip():
+                        logging.warning(f"User {user[0].email} has empty access_token in database")
+                        return None
+                    return user[0]
+                else:
+                    return None
+            except Exception as e:
+                logging.warning(f"load_user got exception {e}")
+                return None
+        else:
+            return None
+
+
+def init_default_admin():
+    # Verify that at least one active admin user exists. If not, create a default one.
+    users = UserService.query(is_superuser=True)
+    if not users:
+        default_admin = {
+            "id": uuid.uuid1().hex,
+            "password": encode_to_base64("admin"),
+            "nickname": "admin",
+            "is_superuser": True,
+            "email": "admin@ragflow.io",
+            "creator": "system",
+            "status": "1",
+        }
+        if not UserService.save(**default_admin):
+            raise AdminException("Can't init admin.", 500)
+    elif not any([u.is_active == ActiveEnum.ACTIVE.value for u in users]):
+        raise AdminException("No active admin. Please update 'is_active' in db manually.", 500)
+
+
+def check_admin_auth(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        user = UserService.filter_by_id(current_user.id)
+        if not user:
+            raise UserNotFoundError(current_user.email)
+        if not user.is_superuser:
+            raise AdminException("Not admin", 403)
+        if user.is_active == ActiveEnum.INACTIVE.value:
+            raise AdminException(f"User {current_user.email} inactive", 403)
+
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def login_admin(email: str, password: str):
+    """
+    :param email: admin email
+    :param password: string before decrypt
+    """
+    users = UserService.query(email=email)
+    if not users:
+        raise UserNotFoundError(email)
+    psw = decrypt(password)
+    user = UserService.query_user(email, psw)
+    if not user:
+        raise AdminException("Email and password do not match!")
+    if not user.is_superuser:
+        raise AdminException("Not admin", 403)
+    if user.is_active == ActiveEnum.INACTIVE.value:
+        raise AdminException(f"User {email} inactive", 403)
+
+    resp = user.to_json()
+    user.access_token = get_uuid()
+    login_user(user)
+    user.update_time = (current_timestamp(),)
+    user.update_date = (datetime_format(datetime.now()),)
+    user.last_login_time = get_format_time()
+    user.save()
+    msg = "Welcome back!"
+    return construct_response(data=resp, auth=user.get_id(), message=msg)
+
+
+def check_admin(username: str, password: str):
+    users = UserService.query(email=username)
+    if not users:
+        logging.info(f"Username: {username} is not registered!")
+        user_info = {
+            "id": uuid.uuid1().hex,
+            "password": encode_to_base64("admin"),
+            "nickname": "admin",
+            "is_superuser": True,
+            "email": "admin@ragflow.io",
+            "creator": "system",
+            "status": "1",
+        }
+        if not UserService.save(**user_info):
+            raise AdminException("Can't init admin.", 500)
+
+    user = UserService.query_user(username, password)
+    if user:
+        return True
+    else:
+        return False
+
+
+def login_verify(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        auth = request.authorization
+        if not auth or 'username' not in auth.parameters or 'password' not in auth.parameters:
+            return jsonify({
+                "code": 401,
+                "message": "Authentication required",
+                "data": None
+            }), 200
+
+        username = auth.parameters['username']
+        password = auth.parameters['password']
+        try:
+            if check_admin(username, password) is False:
+                return jsonify({
+                    "code": 500,
+                    "message": "Access denied",
+                    "data": None
+                }), 200
+        except Exception as e:
+            error_msg = str(e)
+            return jsonify({
+                "code": 500,
+                "message": error_msg
+            }), 200
+
+        return f(*args, **kwargs)
+
+    return decorated

admin/server/config.py

@@ -0,0 +1,317 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+import logging
+import threading
+from enum import Enum
+
+from pydantic import BaseModel
+from typing import Any
+from common.config_utils import read_config
+from urllib.parse import urlparse
+
+
+class ServiceConfigs:
+    configs = dict
+
+    def __init__(self):
+        self.configs = []
+        self.lock = threading.Lock()
+
+
+SERVICE_CONFIGS = ServiceConfigs
+
+
+class ServiceType(Enum):
+    METADATA = "metadata"
+    RETRIEVAL = "retrieval"
+    MESSAGE_QUEUE = "message_queue"
+    RAGFLOW_SERVER = "ragflow_server"
+    TASK_EXECUTOR = "task_executor"
+    FILE_STORE = "file_store"
+
+
+class BaseConfig(BaseModel):
+    id: int
+    name: str
+    host: str
+    port: int
+    service_type: str
+    detail_func_name: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {'id': self.id, 'name': self.name, 'host': self.host, 'port': self.port,
+                'service_type': self.service_type}
+
+
+class MetaConfig(BaseConfig):
+    meta_type: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        extra_dict = result['extra'].copy()
+        extra_dict['meta_type'] = self.meta_type
+        result['extra'] = extra_dict
+        return result
+
+
+class MySQLConfig(MetaConfig):
+    username: str
+    password: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        extra_dict = result['extra'].copy()
+        extra_dict['username'] = self.username
+        extra_dict['password'] = self.password
+        result['extra'] = extra_dict
+        return result
+
+
+class PostgresConfig(MetaConfig):
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        return result
+
+
+class RetrievalConfig(BaseConfig):
+    retrieval_type: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        extra_dict = result['extra'].copy()
+        extra_dict['retrieval_type'] = self.retrieval_type
+        result['extra'] = extra_dict
+        return result
+
+
+class InfinityConfig(RetrievalConfig):
+    db_name: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        extra_dict = result['extra'].copy()
+        extra_dict['db_name'] = self.db_name
+        result['extra'] = extra_dict
+        return result
+
+
+class ElasticsearchConfig(RetrievalConfig):
+    username: str
+    password: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        extra_dict = result['extra'].copy()
+        extra_dict['username'] = self.username
+        extra_dict['password'] = self.password
+        result['extra'] = extra_dict
+        return result
+
+
+class MessageQueueConfig(BaseConfig):
+    mq_type: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        extra_dict = result['extra'].copy()
+        extra_dict['mq_type'] = self.mq_type
+        result['extra'] = extra_dict
+        return result
+
+
+class RedisConfig(MessageQueueConfig):
+    database: int
+    password: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        extra_dict = result['extra'].copy()
+        extra_dict['database'] = self.database
+        extra_dict['password'] = self.password
+        result['extra'] = extra_dict
+        return result
+
+
+class RabbitMQConfig(MessageQueueConfig):
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        return result
+
+
+class RAGFlowServerConfig(BaseConfig):
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        return result
+
+
+class TaskExecutorConfig(BaseConfig):
+    message_queue_type: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        result['extra']['message_queue_type'] = self.message_queue_type
+        return result
+
+
+class FileStoreConfig(BaseConfig):
+    store_type: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        extra_dict = result['extra'].copy()
+        extra_dict['store_type'] = self.store_type
+        result['extra'] = extra_dict
+        return result
+
+
+class MinioConfig(FileStoreConfig):
+    user: str
+    password: str
+
+    def to_dict(self) -> dict[str, Any]:
+        result = super().to_dict()
+        if 'extra' not in result:
+            result['extra'] = dict()
+        extra_dict = result['extra'].copy()
+        extra_dict['user'] = self.user
+        extra_dict['password'] = self.password
+        result['extra'] = extra_dict
+        return result
+
+
+def load_configurations(config_path: str) -> list[BaseConfig]:
+    raw_configs = read_config(config_path)
+    configurations = []
+    ragflow_count = 0
+    id_count = 0
+    for k, v in raw_configs.items():
+        match (k):
+            case "ragflow":
+                name: str = f'ragflow_{ragflow_count}'
+                host: str = v['host']
+                http_port: int = v['http_port']
+                config = RAGFlowServerConfig(id=id_count, name=name, host=host, port=http_port,
+                                             service_type="ragflow_server",
+                                             detail_func_name="check_ragflow_server_alive")
+                configurations.append(config)
+                id_count += 1
+            case "es":
+                name: str = 'elasticsearch'
+                url = v['hosts']
+                parsed = urlparse(url)
+                host: str = parsed.hostname
+                port: int = parsed.port
+                username: str = v.get('username')
+                password: str = v.get('password')
+                config = ElasticsearchConfig(id=id_count, name=name, host=host, port=port, service_type="retrieval",
+                                             retrieval_type="elasticsearch",
+                                             username=username, password=password,
+                                             detail_func_name="get_es_cluster_stats")
+                configurations.append(config)
+                id_count += 1
+
+            case "infinity":
+                name: str = 'infinity'
+                url = v['uri']
+                parts = url.split(':', 1)
+                host = parts[0]
+                port = int(parts[1])
+                database: str = v.get('db_name', 'default_db')
+                config = InfinityConfig(id=id_count, name=name, host=host, port=port, service_type="retrieval",
+                                        retrieval_type="infinity",
+                                        db_name=database, detail_func_name="get_infinity_status")
+                configurations.append(config)
+                id_count += 1
+            case "minio":
+                name: str = 'minio'
+                url = v['host']
+                parts = url.split(':', 1)
+                host = parts[0]
+                port = int(parts[1])
+                user = v.get('user')
+                password = v.get('password')
+                config = MinioConfig(id=id_count, name=name, host=host, port=port, user=user, password=password,
+                                     service_type="file_store",
+                                     store_type="minio", detail_func_name="check_minio_alive")
+                configurations.append(config)
+                id_count += 1
+            case "redis":
+                name: str = 'redis'
+                url = v['host']
+                parts = url.split(':', 1)
+                host = parts[0]
+                port = int(parts[1])
+                password = v.get('password')
+                db: int = v.get('db')
+                config = RedisConfig(id=id_count, name=name, host=host, port=port, password=password, database=db,
+                                     service_type="message_queue", mq_type="redis", detail_func_name="get_redis_info")
+                configurations.append(config)
+                id_count += 1
+            case "mysql":
+                name: str = 'mysql'
+                host: str = v.get('host')
+                port: int = v.get('port')
+                username = v.get('user')
+                password = v.get('password')
+                config = MySQLConfig(id=id_count, name=name, host=host, port=port, username=username, password=password,
+                                     service_type="meta_data", meta_type="mysql", detail_func_name="get_mysql_status")
+                configurations.append(config)
+                id_count += 1
+            case "admin":
+                pass
+            case "task_executor":
+                name: str = 'task_executor'
+                host: str = v.get('host', '')
+                port: int = v.get('port', 0)
+                message_queue_type: str = v.get('message_queue_type')
+                config = TaskExecutorConfig(id=id_count, name=name, host=host, port=port, message_queue_type=message_queue_type,
+                                            service_type="task_executor", detail_func_name="check_task_executor_alive")
+                configurations.append(config)
+                id_count += 1
+            case _:
+                logging.warning(f"Unknown configuration key: {k}")
+                continue
+
+    return configurations

admin/server/exceptions.py

@@ -0,0 +1,17 @@
+class AdminException(Exception):
+    def __init__(self, message, code=400):
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+class UserNotFoundError(AdminException):
+    def __init__(self, username):
+        super().__init__(f"User '{username}' not found", 404)
+
+class UserAlreadyExistsError(AdminException):
+    def __init__(self, username):
+        super().__init__(f"User '{username}' already exists", 409)
+
+class CannotDeleteAdminError(AdminException):
+    def __init__(self):
+        super().__init__("Cannot delete admin account", 403)

admin/server/responses.py

@@ -0,0 +1,34 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+from flask import jsonify
+
+
+def success_response(data=None, message="Success", code=0):
+    return jsonify({
+        "code": code,
+        "message": message,
+        "data": data
+    }), 200
+
+
+def error_response(message="Error", code=-1, data=None):
+    return jsonify({
+        "code": code,
+        "message": message,
+        "data": data
+    }), 400

admin/server/roles.py

@@ -0,0 +1,76 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import logging
+
+from typing import Dict, Any
+
+from api.common.exceptions import AdminException
+
+
+class RoleMgr:
+    @staticmethod
+    def create_role(role_name: str, description: str):
+        error_msg = f"not implement: create role: {role_name}, description: {description}"
+        logging.error(error_msg)
+        raise AdminException(error_msg)
+
+    @staticmethod
+    def update_role_description(role_name: str, description: str) -> Dict[str, Any]:
+        error_msg = f"not implement: update role: {role_name} with description: {description}"
+        logging.error(error_msg)
+        raise AdminException(error_msg)
+
+    @staticmethod
+    def delete_role(role_name: str) -> Dict[str, Any]:
+        error_msg = f"not implement: drop role: {role_name}"
+        logging.error(error_msg)
+        raise AdminException(error_msg)
+
+    @staticmethod
+    def list_roles() -> Dict[str, Any]:
+        error_msg = "not implement: list roles"
+        logging.error(error_msg)
+        raise AdminException(error_msg)
+
+    @staticmethod
+    def get_role_permission(role_name: str) -> Dict[str, Any]:
+        error_msg = f"not implement: show role {role_name}"
+        logging.error(error_msg)
+        raise AdminException(error_msg)
+
+    @staticmethod
+    def grant_role_permission(role_name: str, actions: list, resource: str) -> Dict[str, Any]:
+        error_msg = f"not implement: grant role {role_name} actions: {actions} on {resource}"
+        logging.error(error_msg)
+        raise AdminException(error_msg)
+
+    @staticmethod
+    def revoke_role_permission(role_name: str, actions: list, resource: str) -> Dict[str, Any]:
+        error_msg = f"not implement: revoke role {role_name} actions: {actions} on {resource}"
+        logging.error(error_msg)
+        raise AdminException(error_msg)
+
+    @staticmethod
+    def update_user_role(user_name: str, role_name: str) -> Dict[str, Any]:
+        error_msg = f"not implement: update user role: {user_name} to role {role_name}"
+        logging.error(error_msg)
+        raise AdminException(error_msg)
+
+    @staticmethod
+    def get_user_permission(user_name: str) -> Dict[str, Any]:
+        error_msg = f"not implement: get user permission: {user_name}"
+        logging.error(error_msg)
+        raise AdminException(error_msg)

admin/server/routes.py

@@ -0,0 +1,382 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import secrets
+
+from flask import Blueprint, request
+from flask_login import current_user, logout_user, login_required
+
+from auth import login_verify, login_admin, check_admin_auth
+from responses import success_response, error_response
+from services import UserMgr, ServiceMgr, UserServiceMgr
+from roles import RoleMgr
+from api.common.exceptions import AdminException
+from common.versions import get_ragflow_version
+
+admin_bp = Blueprint('admin', __name__, url_prefix='/api/v1/admin')
+
+
+@admin_bp.route('/login', methods=['POST'])
+def login():
+    if not request.json:
+        return error_response('Authorize admin failed.' ,400)
+    try:
+        email = request.json.get("email", "")
+        password = request.json.get("password", "")
+        return login_admin(email, password)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/logout', methods=['GET'])
+@login_required
+def logout():
+    try:
+        current_user.access_token = f"INVALID_{secrets.token_hex(16)}"
+        current_user.save()
+        logout_user()
+        return success_response(True)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/auth', methods=['GET'])
+@login_verify
+def auth_admin():
+    try:
+        return success_response(None, "Admin is authorized", 0)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/users', methods=['GET'])
+@login_required
+@check_admin_auth
+def list_users():
+    try:
+        users = UserMgr.get_all_users()
+        return success_response(users, "Get all users", 0)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/users', methods=['POST'])
+@login_required
+@check_admin_auth
+def create_user():
+    try:
+        data = request.get_json()
+        if not data or 'username' not in data or 'password' not in data:
+            return error_response("Username and password are required", 400)
+
+        username = data['username']
+        password = data['password']
+        role = data.get('role', 'user')
+
+        res = UserMgr.create_user(username, password, role)
+        if res["success"]:
+            user_info = res["user_info"]
+            user_info.pop("password")  # do not return password
+            return success_response(user_info, "User created successfully")
+        else:
+            return error_response("create user failed")
+
+    except AdminException as e:
+        return error_response(e.message, e.code)
+    except Exception as e:
+        return error_response(str(e))
+
+
+@admin_bp.route('/users/<username>', methods=['DELETE'])
+@login_required
+@check_admin_auth
+def delete_user(username):
+    try:
+        res = UserMgr.delete_user(username)
+        if res["success"]:
+            return success_response(None, res["message"])
+        else:
+            return error_response(res["message"])
+
+    except AdminException as e:
+        return error_response(e.message, e.code)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/users/<username>/password', methods=['PUT'])
+@login_required
+@check_admin_auth
+def change_password(username):
+    try:
+        data = request.get_json()
+        if not data or 'new_password' not in data:
+            return error_response("New password is required", 400)
+
+        new_password = data['new_password']
+        msg = UserMgr.update_user_password(username, new_password)
+        return success_response(None, msg)
+
+    except AdminException as e:
+        return error_response(e.message, e.code)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/users/<username>/activate', methods=['PUT'])
+@login_required
+@check_admin_auth
+def alter_user_activate_status(username):
+    try:
+        data = request.get_json()
+        if not data or 'activate_status' not in data:
+            return error_response("Activation status is required", 400)
+        activate_status = data['activate_status']
+        msg = UserMgr.update_user_activate_status(username, activate_status)
+        return success_response(None, msg)
+    except AdminException as e:
+        return error_response(e.message, e.code)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/users/<username>', methods=['GET'])
+@login_required
+@check_admin_auth
+def get_user_details(username):
+    try:
+        user_details = UserMgr.get_user_details(username)
+        return success_response(user_details)
+
+    except AdminException as e:
+        return error_response(e.message, e.code)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/users/<username>/datasets', methods=['GET'])
+@login_required
+@check_admin_auth
+def get_user_datasets(username):
+    try:
+        datasets_list = UserServiceMgr.get_user_datasets(username)
+        return success_response(datasets_list)
+
+    except AdminException as e:
+        return error_response(e.message, e.code)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/users/<username>/agents', methods=['GET'])
+@login_required
+@check_admin_auth
+def get_user_agents(username):
+    try:
+        agents_list = UserServiceMgr.get_user_agents(username)
+        return success_response(agents_list)
+
+    except AdminException as e:
+        return error_response(e.message, e.code)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/services', methods=['GET'])
+@login_required
+@check_admin_auth
+def get_services():
+    try:
+        services = ServiceMgr.get_all_services()
+        return success_response(services, "Get all services", 0)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/service_types/<service_type>', methods=['GET'])
+@login_required
+@check_admin_auth
+def get_services_by_type(service_type_str):
+    try:
+        services = ServiceMgr.get_services_by_type(service_type_str)
+        return success_response(services)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/services/<service_id>', methods=['GET'])
+@login_required
+@check_admin_auth
+def get_service(service_id):
+    try:
+        services = ServiceMgr.get_service_details(service_id)
+        return success_response(services)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/services/<service_id>', methods=['DELETE'])
+@login_required
+@check_admin_auth
+def shutdown_service(service_id):
+    try:
+        services = ServiceMgr.shutdown_service(service_id)
+        return success_response(services)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/services/<service_id>', methods=['PUT'])
+@login_required
+@check_admin_auth
+def restart_service(service_id):
+    try:
+        services = ServiceMgr.restart_service(service_id)
+        return success_response(services)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/roles', methods=['POST'])
+@login_required
+@check_admin_auth
+def create_role():
+    try:
+        data = request.get_json()
+        if not data or 'role_name' not in data:
+            return error_response("Role name is required", 400)
+        role_name: str = data['role_name']
+        description: str = data['description']
+        res = RoleMgr.create_role(role_name, description)
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/roles/<role_name>', methods=['PUT'])
+@login_required
+@check_admin_auth
+def update_role(role_name: str):
+    try:
+        data = request.get_json()
+        if not data or 'description' not in data:
+            return error_response("Role description is required", 400)
+        description: str = data['description']
+        res = RoleMgr.update_role_description(role_name, description)
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/roles/<role_name>', methods=['DELETE'])
+@login_required
+@check_admin_auth
+def delete_role(role_name: str):
+    try:
+        res = RoleMgr.delete_role(role_name)
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/roles', methods=['GET'])
+@login_required
+@check_admin_auth
+def list_roles():
+    try:
+        res = RoleMgr.list_roles()
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/roles/<role_name>/permission', methods=['GET'])
+@login_required
+@check_admin_auth
+def get_role_permission(role_name: str):
+    try:
+        res = RoleMgr.get_role_permission(role_name)
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/roles/<role_name>/permission', methods=['POST'])
+@login_required
+@check_admin_auth
+def grant_role_permission(role_name: str):
+    try:
+        data = request.get_json()
+        if not data or 'actions' not in data or 'resource' not in data:
+            return error_response("Permission is required", 400)
+        actions: list = data['actions']
+        resource: str = data['resource']
+        res = RoleMgr.grant_role_permission(role_name, actions, resource)
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/roles/<role_name>/permission', methods=['DELETE'])
+@login_required
+@check_admin_auth
+def revoke_role_permission(role_name: str):
+    try:
+        data = request.get_json()
+        if not data or 'actions' not in data or 'resource' not in data:
+            return error_response("Permission is required", 400)
+        actions: list = data['actions']
+        resource: str = data['resource']
+        res = RoleMgr.revoke_role_permission(role_name, actions, resource)
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/users/<user_name>/role', methods=['PUT'])
+@login_required
+@check_admin_auth
+def update_user_role(user_name: str):
+    try:
+        data = request.get_json()
+        if not data or 'role_name' not in data:
+            return error_response("Role name is required", 400)
+        role_name: str = data['role_name']
+        res = RoleMgr.update_user_role(user_name, role_name)
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+
+@admin_bp.route('/users/<user_name>/permission', methods=['GET'])
+@login_required
+@check_admin_auth
+def get_user_permission(user_name: str):
+    try:
+        res = RoleMgr.get_user_permission(user_name)
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)
+
+@admin_bp.route('/version', methods=['GET'])
+@login_required
+@check_admin_auth
+def show_version():
+    try:
+        res = {"version": get_ragflow_version()}
+        return success_response(res)
+    except Exception as e:
+        return error_response(str(e), 500)

admin/server/services.py

@@ -0,0 +1,231 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+import re
+from werkzeug.security import check_password_hash
+from common.constants import ActiveEnum
+from api.db.services import UserService
+from api.db.joint_services.user_account_service import create_new_user, delete_user_data
+from api.db.services.canvas_service import UserCanvasService
+from api.db.services.user_service import TenantService
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.utils.crypt import decrypt
+from api.utils import health_utils
+
+from api.common.exceptions import AdminException, UserAlreadyExistsError, UserNotFoundError
+from config import SERVICE_CONFIGS
+
+
+class UserMgr:
+    @staticmethod
+    def get_all_users():
+        users = UserService.get_all_users()
+        result = []
+        for user in users:
+            result.append({
+                'email': user.email,
+                'nickname': user.nickname,
+                'create_date': user.create_date,
+                'is_active': user.is_active,
+                'is_superuser': user.is_superuser,
+            })
+        return result
+
+    @staticmethod
+    def get_user_details(username):
+        # use email to query
+        users = UserService.query_user_by_email(username)
+        result = []
+        for user in users:
+            result.append({
+                'avatar': user.avatar,
+                'email': user.email,
+                'language': user.language,
+                'last_login_time': user.last_login_time,
+                'is_active': user.is_active,
+                'is_anonymous': user.is_anonymous,
+                'login_channel': user.login_channel,
+                'status': user.status,
+                'is_superuser': user.is_superuser,
+                'create_date': user.create_date,
+                'update_date': user.update_date
+            })
+        return result
+
+    @staticmethod
+    def create_user(username, password, role="user") -> dict:
+        # Validate the email address
+        if not re.match(r"^[\w\._-]+@([\w_-]+\.)+[\w-]{2,}$", username):
+            raise AdminException(f"Invalid email address: {username}!")
+        # Check if the email address is already used
+        if UserService.query(email=username):
+            raise UserAlreadyExistsError(username)
+        # Construct user info data
+        user_info_dict = {
+            "email": username,
+            "nickname": "",  # ask user to edit it manually in settings.
+            "password": decrypt(password),
+            "login_channel": "password",
+            "is_superuser": role == "admin",
+        }
+        return create_new_user(user_info_dict)
+
+    @staticmethod
+    def delete_user(username):
+        # use email to delete
+        user_list = UserService.query_user_by_email(username)
+        if not user_list:
+            raise UserNotFoundError(username)
+        if len(user_list) > 1:
+            raise AdminException(f"Exist more than 1 user: {username}!")
+        usr = user_list[0]
+        return delete_user_data(usr.id)
+
+    @staticmethod
+    def update_user_password(username, new_password) -> str:
+        # use email to find user. check exist and unique.
+        user_list = UserService.query_user_by_email(username)
+        if not user_list:
+            raise UserNotFoundError(username)
+        elif len(user_list) > 1:
+            raise AdminException(f"Exist more than 1 user: {username}!")
+        # check new_password different from old.
+        usr = user_list[0]
+        psw = decrypt(new_password)
+        if check_password_hash(usr.password, psw):
+            return "Same password, no need to update!"
+        # update password
+        UserService.update_user_password(usr.id, psw)
+        return "Password updated successfully!"
+
+    @staticmethod
+    def update_user_activate_status(username, activate_status: str):
+        # use email to find user. check exist and unique.
+        user_list = UserService.query_user_by_email(username)
+        if not user_list:
+            raise UserNotFoundError(username)
+        elif len(user_list) > 1:
+            raise AdminException(f"Exist more than 1 user: {username}!")
+        # check activate status different from new
+        usr = user_list[0]
+        # format activate_status before handle
+        _activate_status = activate_status.lower()
+        target_status = {
+            'on': ActiveEnum.ACTIVE.value,
+            'off': ActiveEnum.INACTIVE.value,
+        }.get(_activate_status)
+        if not target_status:
+            raise AdminException(f"Invalid activate_status: {activate_status}")
+        if target_status == usr.is_active:
+            return f"User activate status is already {_activate_status}!"
+        # update is_active
+        UserService.update_user(usr.id, {"is_active": target_status})
+        return f"Turn {_activate_status} user activate status successfully!"
+
+
+class UserServiceMgr:
+
+    @staticmethod
+    def get_user_datasets(username):
+        # use email to find user.
+        user_list = UserService.query_user_by_email(username)
+        if not user_list:
+            raise UserNotFoundError(username)
+        elif len(user_list) > 1:
+            raise AdminException(f"Exist more than 1 user: {username}!")
+        # find tenants
+        usr = user_list[0]
+        tenants = TenantService.get_joined_tenants_by_user_id(usr.id)
+        tenant_ids = [m["tenant_id"] for m in tenants]
+        # filter permitted kb and owned kb
+        return KnowledgebaseService.get_all_kb_by_tenant_ids(tenant_ids, usr.id)
+
+    @staticmethod
+    def get_user_agents(username):
+        # use email to find user.
+        user_list = UserService.query_user_by_email(username)
+        if not user_list:
+            raise UserNotFoundError(username)
+        elif len(user_list) > 1:
+            raise AdminException(f"Exist more than 1 user: {username}!")
+        # find tenants
+        usr = user_list[0]
+        tenants = TenantService.get_joined_tenants_by_user_id(usr.id)
+        tenant_ids = [m["tenant_id"] for m in tenants]
+        # filter permitted agents and owned agents
+        res = UserCanvasService.get_all_agents_by_tenant_ids(tenant_ids, usr.id)
+        return [{
+            'title': r['title'],
+            'permission': r['permission'],
+            'canvas_category': r['canvas_category'].split('_')[0],
+            'avatar': r['avatar']
+        } for r in res]
+
+
+class ServiceMgr:
+
+    @staticmethod
+    def get_all_services():
+        result = []
+        configs = SERVICE_CONFIGS.configs
+        for service_id, config in enumerate(configs):
+            config_dict = config.to_dict()
+            try:
+                service_detail = ServiceMgr.get_service_details(service_id)
+                if "status" in service_detail:
+                    config_dict['status'] = service_detail['status']
+                else:
+                    config_dict['status'] = 'timeout'
+            except Exception:
+                config_dict['status'] = 'timeout'
+            if not config_dict['host']:
+                config_dict['host'] = '-'
+            if not config_dict['port']:
+                config_dict['port'] = '-'
+            result.append(config_dict)
+        return result
+
+    @staticmethod
+    def get_services_by_type(service_type_str: str):
+        raise AdminException("get_services_by_type: not implemented")
+
+    @staticmethod
+    def get_service_details(service_id: int):
+        service_id = int(service_id)
+        configs = SERVICE_CONFIGS.configs
+        service_config_mapping = {
+            c.id: {
+                'name': c.name,
+                'detail_func_name': c.detail_func_name
+            } for c in configs
+        }
+        service_info = service_config_mapping.get(service_id, {})
+        if not service_info:
+            raise AdminException(f"invalid service_id: {service_id}")
+
+        detail_func = getattr(health_utils, service_info.get('detail_func_name'))
+        res = detail_func()
+        res.update({'service_name': service_info.get('name')})
+        return res
+
+    @staticmethod
+    def shutdown_service(service_id: int):
+        raise AdminException("shutdown_service: not implemented")
+
+    @staticmethod
+    def restart_service(service_id: int):
+        raise AdminException("restart_service: not implemented")

agent/canvas.py

@@ -26,8 +26,10 @@ from typing import Any, Union, Tuple
 from agent.component import component_class
 from agent.component.base import ComponentBase
 from api.db.services.file_service import FileService
-from api.utils import get_uuid, hash_str2int
-from rag.prompts.prompts import chunks_format
+from api.db.services.task_service import has_canceled
+from common.misc_utils import get_uuid, hash_str2int
+from common.exceptions import TaskCanceledException
+from rag.prompts.generator import chunks_format
 from rag.utils.redis_conn import REDIS_CONN
 
 class Graph:
@@ -126,6 +128,7 @@ class Graph:
             self.components[k]["obj"].reset()
         try:
             REDIS_CONN.delete(f"{self.task_id}-logs")
+            REDIS_CONN.delete(f"{self.task_id}-cancel")
         except Exception as e:
             logging.exception(e)
 
@@ -153,6 +156,79 @@ class Graph:
     def get_tenant_id(self):
         return self._tenant_id
 
+    def get_value_with_variable(self,value: str) -> Any:
+        pat = re.compile(r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z0-9_.]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*")
+        out_parts = []
+        last = 0
+
+        for m in pat.finditer(value):
+            out_parts.append(value[last:m.start()])
+            key = m.group(1)
+            v = self.get_variable_value(key)
+            if v is None:
+                rep = ""
+            elif isinstance(v, partial):
+                buf = []
+                for chunk in v():
+                    buf.append(chunk)
+                rep = "".join(buf)
+            elif isinstance(v, str):
+                rep = v
+            else:
+                rep = json.dumps(v, ensure_ascii=False)
+
+            out_parts.append(rep)
+            last = m.end()
+
+        out_parts.append(value[last:])
+        return("".join(out_parts))
+
+    def get_variable_value(self, exp: str) -> Any:
+        exp = exp.strip("{").strip("}").strip(" ").strip("{").strip("}")
+        if exp.find("@") < 0:
+            return self.globals[exp]
+        cpn_id, var_nm = exp.split("@")
+        cpn = self.get_component(cpn_id)
+        if not cpn:
+            raise Exception(f"Can't find variable: '{cpn_id}@{var_nm}'")
+        parts = var_nm.split(".", 1)
+        root_key = parts[0]
+        rest = parts[1] if len(parts) > 1 else ""
+        root_val = cpn["obj"].output(root_key)
+
+        if not rest:
+            return root_val
+        return self.get_variable_param_value(root_val,rest)
+
+    def get_variable_param_value(self, obj: Any, path: str) -> Any:
+        cur = obj
+        if not path:
+            return cur
+        for key in path.split('.'):
+            if cur is None:
+                return None
+            if isinstance(cur, str):
+                try:
+                    cur = json.loads(cur)
+                except Exception:
+                    return None
+            if isinstance(cur, dict):
+                cur = cur.get(key)
+            else:
+                cur = getattr(cur, key, None)
+        return cur
+
+    def is_canceled(self) -> bool:
+        return has_canceled(self.task_id)
+
+    def cancel_task(self) -> bool:
+        try:
+            REDIS_CONN.set(f"{self.task_id}-cancel", "x")
+        except Exception as e:
+            logging.exception(e)
+            return False
+        return True
+
 
 class Canvas(Graph):
 
@@ -177,7 +253,7 @@ class Canvas(Graph):
             "sys.conversation_turns": 0,
             "sys.files": []
         }
-            
+
         self.retrieval = self.dsl["retrieval"]
         self.memory = self.dsl.get("memory", [])
 
@@ -193,20 +269,20 @@ class Canvas(Graph):
             self.history = []
             self.retrieval = []
             self.memory = []
-
         for k in self.globals.keys():
-            if isinstance(self.globals[k], str):
-                self.globals[k] = ""
-            elif isinstance(self.globals[k], int):
-                self.globals[k] = 0
-            elif isinstance(self.globals[k], float):
-                self.globals[k] = 0
-            elif isinstance(self.globals[k], list):
-                self.globals[k] = []
-            elif isinstance(self.globals[k], dict):
-                self.globals[k] = {}
-            else:
-                self.globals[k] = None
+            if k.startswith("sys."):
+                if isinstance(self.globals[k], str):
+                    self.globals[k] = ""
+                elif isinstance(self.globals[k], int):
+                    self.globals[k] = 0
+                elif isinstance(self.globals[k], float):
+                    self.globals[k] = 0
+                elif isinstance(self.globals[k], list):
+                    self.globals[k] = []
+                elif isinstance(self.globals[k], dict):
+                    self.globals[k] = {}
+                else:
+                    self.globals[k] = None
 
     def run(self, **kwargs):
         st = time.perf_counter()
@@ -216,6 +292,14 @@ class Canvas(Graph):
         for k, cpn in self.components.items():
             self.components[k]["obj"].reset(True)
 
+        if kwargs.get("webhook_payload"):
+            for k, cpn in self.components.items():
+                if self.components[k]["obj"].component_name.lower() == "webhook":
+                    for kk, vv in kwargs["webhook_payload"].items():
+                        self.components[k]["obj"].set_output(kk, vv)
+
+            self.components[k]["obj"].reset(True)
+
         for k in kwargs.keys():
             if k in ["query", "user_id", "files"] and kwargs[k]:
                 if k == "files":
@@ -241,18 +325,37 @@ class Canvas(Graph):
             self.path.append("begin")
             self.retrieval.append({"chunks": [], "doc_aggs": []})
 
+        if self.is_canceled():
+            msg = f"Task {self.task_id} has been canceled before starting."
+            logging.info(msg)
+            raise TaskCanceledException(msg)
+
         yield decorate("workflow_started", {"inputs": kwargs.get("inputs")})
         self.retrieval.append({"chunks": {}, "doc_aggs": {}})
 
         def _run_batch(f, t):
+            if self.is_canceled():
+                msg = f"Task {self.task_id} has been canceled during batch execution."
+                logging.info(msg)
+                raise TaskCanceledException(msg)
+
             with ThreadPoolExecutor(max_workers=5) as executor:
                 thr = []
-                for i in range(f, t):
+                i = f
+                while i < t:
                     cpn = self.get_component_obj(self.path[i])
                     if cpn.component_name.lower() in ["begin", "userfillup"]:
                         thr.append(executor.submit(cpn.invoke, inputs=kwargs.get("inputs", {})))
+                        i += 1
                     else:
-                        thr.append(executor.submit(cpn.invoke, **cpn.get_input()))
+                        for _, ele in cpn.get_input_elements().items():
+                            if isinstance(ele, dict) and ele.get("_cpn_id") and ele.get("_cpn_id") not in self.path[:i] and self.path[0].lower().find("userfillup") < 0:
+                                self.path.pop(i)
+                                t -= 1
+                                break
+                        else:
+                            thr.append(executor.submit(cpn.invoke, **cpn.get_input()))
+                            i += 1
                 for t in thr:
                     t.result()
 
@@ -282,7 +385,7 @@ class Canvas(Graph):
                     "thoughts": self.get_component_thoughts(self.path[i])
                 })
             _run_batch(idx, to)
-
+            to = len(self.path)
             # post processing of components invocation
             for i in range(idx, to):
                 cpn = self.get_component(self.path[i])
@@ -377,13 +480,13 @@ class Canvas(Graph):
                 for c in path:
                     o = self.get_component_obj(c)
                     if o.component_name.lower() == "userfillup":
+                        o.invoke()
                         another_inputs.update(o.get_input_elements())
                         if o.get_param("enable_tips"):
-                            tips = o.get_param("tips")
+                            tips = o.output("tips")
                 self.path = path
                 yield decorate("user_inputs", {"inputs": another_inputs, "tips": tips})
                 return
-
         self.path = self.path[:idx]
         if not self.error:
             yield decorate("workflow_finished",
@@ -394,6 +497,14 @@ class Canvas(Graph):
                            "created_at": st,
                        })
             self.history.append(("assistant", self.get_component_obj(self.path[-1]).output()))
+        elif "Task has been canceled" in self.error:
+            yield decorate("workflow_finished",
+                       {
+                           "inputs": kwargs.get("inputs"),
+                           "outputs": "Task has been canceled",
+                           "elapsed_time": time.perf_counter() - st,
+                           "created_at": st,
+                       })
 
     def is_reff(self, exp: str) -> bool:
         exp = exp.strip("{").strip("}")
@@ -406,16 +517,6 @@ class Canvas(Graph):
             return False
         return True
 
-    def get_variable_value(self, exp: str) -> Any:
-        exp = exp.strip("{").strip("}").strip(" ").strip("{").strip("}")
-        if exp.find("@") < 0:
-            return self.globals[exp]
-        cpn_id, var_nm = exp.split("@")
-        cpn = self.get_component(cpn_id)
-        if not cpn:
-            raise Exception(f"Can't find variable: '{cpn_id}@{var_nm}'")
-        return cpn["obj"].output(var_nm)
-
     def get_history(self, window_size):
         convs = []
         if window_size <= 0:
@@ -490,7 +591,8 @@ class Canvas(Graph):
 
         r = self.retrieval[-1]
         for ck in chunks_format({"chunks": chunks}):
-            cid = hash_str2int(ck["id"], 100)
+            cid = hash_str2int(ck["id"], 500)
+            # cid = uuid.uuid5(uuid.NAMESPACE_DNS, ck["id"])
             if cid not in r:
                 r["chunks"][cid] = ck
 

agent/component/__init__.py

@@ -13,7 +13,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-
 import os
 import importlib
 import inspect
@@ -50,9 +49,10 @@ del _package_path, _import_submodules, _extract_classes_from_module
 
 
 def component_class(class_name):
-    for mdl in ["agent.component", "agent.tools", "rag.flow"]:
+    for module_name in ["agent.component", "agent.tools", "rag.flow"]:
         try:
-            return getattr(importlib.import_module(mdl), class_name)
+            return getattr(importlib.import_module(module_name), class_name)
         except Exception:
+            # logging.warning(f"Can't import module: {module_name}, error: {e}")
             pass
     assert False, f"Can't import {class_name}"

agent/component/agent_with_tools.py

@@ -27,10 +27,9 @@ from agent.tools.base import LLMToolPluginCallSession, ToolParamBase, ToolBase,
 from api.db.services.llm_service import LLMBundle
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.mcp_server_service import MCPServerService
-from api.utils.api_utils import timeout
-from rag.prompts import message_fit_in
-from rag.prompts.prompts import next_step, COMPLETE_TASK, analyze_task, \
-    citation_prompt, reflect, rank_memories, kb_prompt, citation_plus, full_question
+from common.connection_utils import timeout
+from rag.prompts.generator import next_step, COMPLETE_TASK, analyze_task, \
+    citation_prompt, reflect, rank_memories, kb_prompt, citation_plus, full_question, message_fit_in
 from rag.utils.mcp_tool_call_conn import MCPToolCallSession, mcp_tool_metadata_to_openai_tool
 from agent.component.llm import LLMParam, LLM
 
@@ -138,8 +137,11 @@ class Agent(LLM, ToolBase):
             res.update(cpn.get_input_form())
         return res
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 20*60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 20*60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Agent processing"):
+            return
+
         if kwargs.get("user_prompt"):
             usr_pmt = ""
             if kwargs.get("reasoning"):
@@ -153,13 +155,20 @@ class Agent(LLM, ToolBase):
             self._param.prompts = [{"role": "user", "content": usr_pmt}]
 
         if not self.tools:
+            if self.check_if_canceled("Agent processing"):
+                return
             return LLM._invoke(self, **kwargs)
 
         prompt, msg, user_defined_prompt = self._prepare_prompt_variables()
 
         downstreams = self._canvas.get_component(self._id)["downstream"] if self._canvas.get_component(self._id) else []
         ex = self.exception_handler()
-        if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not self._param.output_structure and not (ex and ex["goto"]):
+        output_structure=None
+        try:
+            output_structure=self._param.outputs['structured']
+        except Exception:
+            pass
+        if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not output_structure and not (ex and ex["goto"]):
             self.set_output("content", partial(self.stream_output_with_tools, prompt, msg, user_defined_prompt))
             return
 
@@ -167,6 +176,8 @@ class Agent(LLM, ToolBase):
         use_tools = []
         ans = ""
         for delta_ans, tk in self._react_with_tools_streamly(prompt, msg, use_tools, user_defined_prompt):
+            if self.check_if_canceled("Agent processing"):
+                return
             ans += delta_ans
 
         if ans.find("**ERROR**") >= 0:
@@ -187,12 +198,16 @@ class Agent(LLM, ToolBase):
         answer_without_toolcall = ""
         use_tools = []
         for delta_ans,_ in self._react_with_tools_streamly(prompt, msg, use_tools, user_defined_prompt):
+            if self.check_if_canceled("Agent streaming"):
+                return
+
             if delta_ans.find("**ERROR**") >= 0:
                 if self.get_exception_default_value():
                     self.set_output("content", self.get_exception_default_value())
                     yield self.get_exception_default_value()
                 else:
                     self.set_output("_ERROR", delta_ans)
+                    return
             answer_without_toolcall += delta_ans
             yield delta_ans
 
@@ -267,6 +282,8 @@ class Agent(LLM, ToolBase):
             st = timer()
             txt = ""
             for delta_ans in self._gen_citations(entire_txt):
+                if self.check_if_canceled("Agent streaming"):
+                    return
                 yield delta_ans, 0
                 txt += delta_ans
 
@@ -282,6 +299,8 @@ class Agent(LLM, ToolBase):
         task_desc = analyze_task(self.chat_mdl, prompt, user_request, tool_metas, user_defined_prompt)
         self.callback("analyze_task", {}, task_desc, elapsed_time=timer()-st)
         for _ in range(self._param.max_rounds + 1):
+            if self.check_if_canceled("Agent streaming"):
+                return
             response, tk = next_step(self.chat_mdl, hist, tool_metas, task_desc, user_defined_prompt)
             # self.callback("next_step", {}, str(response)[:256]+"...")
             token_count += tk
@@ -329,6 +348,8 @@ Instructions:
 6. Focus on delivering VALUE with the information already gathered
 Respond immediately with your final comprehensive answer.
         """
+        if self.check_if_canceled("Agent final instruction"):
+            return
         append_user_content(hist, final_instruction)
 
         for txt, tkcnt in complete():
@@ -347,3 +368,11 @@ Respond immediately with your final comprehensive answer.
 
         return "Error occurred."
 
+    def reset(self, temp=False):
+        """
+        Reset all tools if they have a reset method. This avoids errors for tools like MCPToolCallSession.
+        """
+        for k, cpn in self.tools.items():
+            if hasattr(cpn, "reset") and callable(cpn.reset):
+                cpn.reset()
+

agent/component/base.py

@@ -25,7 +25,7 @@ from typing import Any, List, Union
 import pandas as pd
 import trio
 from agent import settings
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 _FEEDED_DEPRECATED_PARAMS = "_feeded_deprecated_params"
@@ -244,7 +244,7 @@ class ComponentParamBase(ABC):
 
                 if not value_legal:
                     raise ValueError(
-                        "Plase check runtime conf, {} = {} does not match user-parameter restriction".format(
+                        "Please check runtime conf, {} = {} does not match user-parameter restriction".format(
                             variable, value
                         )
                     )
@@ -393,7 +393,7 @@ class ComponentParamBase(ABC):
 class ComponentBase(ABC):
     component_name: str
     thread_limiter = trio.CapacityLimiter(int(os.environ.get('MAX_CONCURRENT_CHATS', 10)))
-    variable_ref_patt = r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z:0-9_.-]+|sys\.[a-z_]+)\} *\}*"
+    variable_ref_patt = r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z0-9_.]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*"
 
     def __str__(self):
         """
@@ -417,6 +417,20 @@ class ComponentBase(ABC):
         self._param = param
         self._param.check()
 
+    def is_canceled(self) -> bool:
+        return self._canvas.is_canceled()
+
+    def check_if_canceled(self, message: str = "") -> bool:
+        if self.is_canceled():
+            task_id = getattr(self._canvas, 'task_id', 'unknown')
+            log_message = f"Task {task_id} has been canceled"
+            if message:
+                log_message += f" during {message}"
+            logging.info(log_message)
+            self.set_output("_ERROR", "Task has been canceled")
+            return True
+        return False
+
     def invoke(self, **kwargs) -> dict[str, Any]:
         self.set_output("_created_time", time.perf_counter())
         try:
@@ -431,7 +445,7 @@ class ComponentBase(ABC):
         self.set_output("_elapsed_time", time.perf_counter() - self.output("_created_time"))
         return self.output()
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
     def _invoke(self, **kwargs):
         raise NotImplementedError()
 
@@ -514,6 +528,7 @@ class ComponentBase(ABC):
     def get_param(self, name):
         if hasattr(self._param, name):
             return getattr(self._param, name)
+        return None
 
     def debug(self, **kwargs):
         return self._invoke(**kwargs)
@@ -521,7 +536,7 @@ class ComponentBase(ABC):
     def get_parent(self) -> Union[object, None]:
         pid = self._canvas.get_component(self._id).get("parent_id")
         if not pid:
-            return
+            return None
         return self._canvas.get_component(pid)["obj"]
 
     def get_upstream(self) -> List[str]:
@@ -546,7 +561,7 @@ class ComponentBase(ABC):
 
     def exception_handler(self):
         if not self._param.exception_method:
-            return
+            return None
         return {
             "goto": self._param.exception_goto,
             "default_value": self._param.exception_default_value

agent/component/begin.py

@@ -37,7 +37,13 @@ class Begin(UserFillUp):
     component_name = "Begin"
 
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Begin processing"):
+            return
+
         for k, v in kwargs.get("inputs", {}).items():
+            if self.check_if_canceled("Begin processing"):
+                return
+
             if isinstance(v, dict) and v.get("type", "").lower().find("file") >=0:
                 if v.get("optional") and v.get("value", None) is None:
                     v = None

agent/component/categorize.py

@@ -18,17 +18,17 @@ import os
 import re
 from abc import ABC
 
-from api.db import LLMType
+from common.constants import LLMType
 from api.db.services.llm_service import LLMBundle
 from agent.component.llm import LLMParam, LLM
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 from rag.llm.chat_model import ERROR_PREFIX
 
 
 class CategorizeParam(LLMParam):
 
     """
-    Define the Categorize component parameters.
+    Define the categorize component parameters.
     """
     def __init__(self):
         super().__init__()
@@ -80,7 +80,7 @@ Here's description of each category:
  - Prioritize the most specific applicable category
  - Return only the category name without explanations
  - Use "Other" only when no other category fits
- 
+
  """.format(
             "\n - ".join(list(self.category_description.keys())),
             "\n".join(descriptions)
@@ -96,8 +96,11 @@ Here's description of each category:
 class Categorize(LLM, ABC):
     component_name = "Categorize"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Categorize processing"):
+            return
+
         msg = self._canvas.get_history(self._param.message_history_window_size)
         if not msg:
             msg = [{"role": "user", "content": ""}]
@@ -112,12 +115,20 @@ class Categorize(LLM, ABC):
 
         user_prompt = """
 ---- Real Data ----
-{} → 
+{} →
 """.format(" | ".join(["{}: \"{}\"".format(c["role"].upper(), re.sub(r"\n", "", c["content"], flags=re.DOTALL)) for c in msg]))
+
+        if self.check_if_canceled("Categorize processing"):
+            return
+
         ans = chat_mdl.chat(self._param.sys_prompt, [{"role": "user", "content": user_prompt}], self._param.gen_conf())
         logging.info(f"input: {user_prompt}, answer: {str(ans)}")
         if ERROR_PREFIX in ans:
             raise Exception(ans)
+
+        if self.check_if_canceled("Categorize processing"):
+            return
+
         # Count the number of times each category appears in the answer.
         category_counts = {}
         for c in self._param.category_description.keys():
@@ -134,4 +145,4 @@ class Categorize(LLM, ABC):
         self.set_output("_next", cpn_ids)
 
     def thoughts(self) -> str:
-        return "Which should it falls into {}? ...".format(",".join([f"`{c}`" for c, _ in self._param.category_description.items()]))
+        return "Which should it falls into {}? ...".format(",".join([f"`{c}`" for c, _ in self._param.category_description.items()]))

agent/component/data_operations.py

@@ -0,0 +1,203 @@
+from abc import ABC
+import ast
+import os
+from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.api_utils import timeout
+
+class DataOperationsParam(ComponentParamBase):
+    """
+    Define the Data Operations component parameters.
+    """
+    def __init__(self):
+        super().__init__()
+        self.query = []
+        self.operations = "literal_eval"
+        self.select_keys = []
+        self.filter_values=[]
+        self.updates=[]
+        self.remove_keys=[]
+        self.rename_keys=[]
+        self.outputs = {
+            "result": {
+                "value": [],
+                "type": "Array of Object"
+            }
+        }
+    
+    def check(self):
+        self.check_valid_value(self.operations, "Support operations", ["select_keys", "literal_eval","combine","filter_values","append_or_update","remove_keys","rename_keys"])
+    
+    
+
+class DataOperations(ComponentBase,ABC):
+    component_name = "DataOperations"
+
+    def get_input_form(self) -> dict[str, dict]:
+        return {
+            k: {"name": o.get("name", ""), "type": "line"}
+            for input_item in (self._param.query or [])
+            for k, o in self.get_input_elements_from_text(input_item).items()
+        }
+
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
+    def _invoke(self, **kwargs):
+        self.input_objects=[]
+        inputs = getattr(self._param, "query", None)
+        if not isinstance(inputs, (list, tuple)):
+            inputs = [inputs]
+        for input_ref in inputs:
+            input_object=self._canvas.get_variable_value(input_ref)
+            self.set_input_value(input_ref, input_object)
+            if input_object is None:
+                continue
+            if isinstance(input_object,dict):
+                self.input_objects.append(input_object)
+            elif isinstance(input_object,list):
+                self.input_objects.extend(x for x in input_object if isinstance(x, dict))
+            else:
+                continue
+        if self._param.operations == "select_keys":
+            self._select_keys()
+        elif self._param.operations == "recursive_eval":
+            self._literal_eval()
+        elif self._param.operations == "combine":
+            self._combine()
+        elif self._param.operations == "filter_values":
+            self._filter_values()
+        elif self._param.operations == "append_or_update":
+            self._append_or_update()
+        elif self._param.operations == "remove_keys":
+            self._remove_keys()
+        else:
+            self._rename_keys()
+    
+    def _select_keys(self):
+        filter_criteria: list[str] = self._param.select_keys
+        results = [{key: value for key, value in data_dict.items() if key in filter_criteria} for data_dict in self.input_objects]
+        self.set_output("result", results)
+
+
+    def _recursive_eval(self, data):
+        if isinstance(data, dict):
+            return {k: self.recursive_eval(v) for k, v in data.items()}
+        if isinstance(data, list):
+            return [self.recursive_eval(item) for item in data]
+        if isinstance(data, str):
+            try:
+                if (
+                    data.strip().startswith(("{", "[", "(", "'", '"'))
+                    or data.strip().lower() in ("true", "false", "none")
+                    or data.strip().replace(".", "").isdigit()
+                ):
+                    return ast.literal_eval(data)
+            except (ValueError, SyntaxError, TypeError, MemoryError):
+                return data
+            else:
+                return data
+        return data
+    
+    def _literal_eval(self):
+        self.set_output("result", self._recursive_eval(self.input_objects))
+
+    def _combine(self):
+        result={}
+        for obj in self.input_objects:
+            for key, value in obj.items():
+                if key not in result:
+                    result[key] = value
+                elif isinstance(result[key], list):
+                    if isinstance(value, list):
+                        result[key].extend(value)
+                    else:
+                        result[key].append(value)
+                else:
+                    result[key] = (
+                        [result[key], value] if not isinstance(value, list) else [result[key], *value]
+                    )
+        self.set_output("result", result)
+    
+    def norm(self,v):
+        s = "" if v is None else str(v)
+        return s
+    
+    def match_rule(self, obj, rule):
+        key = rule.get("key")
+        op = (rule.get("operator") or "equals").lower()
+        target = self.norm(rule.get("value"))
+        target = self._canvas.get_value_with_variable(target) or target
+        if key not in obj:
+            return False
+        val = obj.get(key, None)
+        v = self.norm(val)
+        if op == "=":
+            return v == target
+        if op == "≠":
+            return v != target
+        if op == "contains":
+            return target in v
+        if op == "start with":
+            return v.startswith(target)
+        if op == "end with":
+            return v.endswith(target)
+        return False
+        
+    def _filter_values(self):
+        results=[]
+        rules = (getattr(self._param, "filter_values", None) or [])
+        for obj in self.input_objects:
+            if not rules:
+                results.append(obj)
+                continue
+            if all(self.match_rule(obj, r) for r in rules):
+                results.append(obj)
+        self.set_output("result", results)
+            
+                
+    def _append_or_update(self):
+        results=[]
+        updates = getattr(self._param, "updates", []) or [] 
+        for obj in self.input_objects:
+            new_obj = dict(obj)
+            for item in updates:
+                if not isinstance(item, dict):
+                    continue
+                k = (item.get("key") or "").strip()
+                if not k:
+                    continue
+                new_obj[k] = self._canvas.get_value_with_variable(item.get("value")) or item.get("value")
+            results.append(new_obj)
+        self.set_output("result", results)
+
+    def _remove_keys(self):
+        results = []
+        remove_keys = getattr(self._param, "remove_keys", []) or []
+
+        for obj in (self.input_objects or []):
+            new_obj = dict(obj)
+            for k in remove_keys:
+                if not isinstance(k, str):
+                    continue
+                new_obj.pop(k, None)
+            results.append(new_obj)
+        self.set_output("result", results)
+
+    def _rename_keys(self):
+        results = []
+        rename_pairs = getattr(self._param, "rename_keys", []) or []
+
+        for obj in (self.input_objects or []):
+            new_obj = dict(obj)
+            for pair in rename_pairs:
+                if not isinstance(pair, dict):
+                    continue
+                old = (pair.get("old_key") or "").strip()
+                new = (pair.get("new_key") or "").strip()
+                if not old or not new or old == new:
+                    continue
+                if old in new_obj:
+                    new_obj[new] = new_obj.pop(old)
+            results.append(new_obj)
+        self.set_output("result", results)
+
+    def thoughts(self) -> str:
+        return "DataOperation in progress"

agent/component/fillup.py

@@ -13,7 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-from agent.component.base import ComponentBase, ComponentParamBase
+import json
+import re
+from functools import partial
+
+from agent.component.base import ComponentParamBase, ComponentBase
 
 
 class UserFillUpParam(ComponentParamBase):
@@ -31,10 +35,35 @@ class UserFillUp(ComponentBase):
     component_name = "UserFillUp"
 
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("UserFillUp processing"):
+            return
+
+        if self._param.enable_tips:
+            content = self._param.tips
+            for k, v in self.get_input_elements_from_text(self._param.tips).items():
+                v = v["value"]
+                ans = ""
+                if isinstance(v, partial):
+                    for t in v():
+                        ans += t
+                elif isinstance(v, list):
+                    ans = ",".join([str(vv) for vv in v])
+                elif not isinstance(v, str):
+                    try:
+                        ans = json.dumps(v, ensure_ascii=False)
+                    except Exception:
+                        pass
+                else:
+                    ans = v
+                if not ans:
+                    ans = ""
+                content = re.sub(r"\{%s\}"%k, ans, content)
+
+            self.set_output("tips", content)
         for k, v in kwargs.get("inputs", {}).items():
+            if self.check_if_canceled("UserFillUp processing"):
+                return
             self.set_output(k, v)
 
     def thoughts(self) -> str:
         return "Waiting for your input..."
-
-

agent/component/invoke.py

@@ -19,11 +19,12 @@ import os
 import re
 import time
 from abc import ABC
+
 import requests
 
-from api.utils.api_utils import timeout
-from deepdoc.parser import HtmlParser
 from agent.component.base import ComponentBase, ComponentParamBase
+from common.connection_utils import timeout
+from deepdoc.parser import HtmlParser
 
 
 class InvokeParam(ComponentParamBase):
@@ -43,18 +44,21 @@ class InvokeParam(ComponentParamBase):
         self.datatype = "json"  # New parameter to determine data posting type
 
     def check(self):
-        self.check_valid_value(self.method.lower(), "Type of content from the crawler", ['get', 'post', 'put'])
+        self.check_valid_value(self.method.lower(), "Type of content from the crawler", ["get", "post", "put"])
         self.check_empty(self.url, "End point URL")
         self.check_positive_integer(self.timeout, "Timeout time in second")
         self.check_boolean(self.clean_html, "Clean HTML")
-        self.check_valid_value(self.datatype.lower(), "Data post type", ['json', 'formdata'])  # Check for valid datapost value
+        self.check_valid_value(self.datatype.lower(), "Data post type", ["json", "formdata"])  # Check for valid datapost value
 
 
 class Invoke(ComponentBase, ABC):
     component_name = "Invoke"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 3))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 3)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Invoke processing"):
+            return
+
         args = {}
         for para in self._param.variables:
             if para.get("value"):
@@ -63,6 +67,18 @@ class Invoke(ComponentBase, ABC):
                 args[para["key"]] = self._canvas.get_variable_value(para["ref"])
 
         url = self._param.url.strip()
+
+        def replace_variable(match):
+            var_name = match.group(1)
+            try:
+                value = self._canvas.get_variable_value(var_name)
+                return str(value or "")
+            except Exception:
+                return ""
+
+        # {base_url} or {component_id@variable_name}
+        url = re.sub(r"\{([a-zA-Z_][a-zA-Z0-9_.@-]*)\}", replace_variable, url)
+
         if url.find("http") != 0:
             url = "http://" + url
 
@@ -75,52 +91,35 @@ class Invoke(ComponentBase, ABC):
             proxies = {"http": self._param.proxy, "https": self._param.proxy}
 
         last_e = ""
-        for _ in range(self._param.max_retries+1):
+        for _ in range(self._param.max_retries + 1):
+            if self.check_if_canceled("Invoke processing"):
+                return
+
             try:
-                if method == 'get':
-                    response = requests.get(url=url,
-                                            params=args,
-                                            headers=headers,
-                                            proxies=proxies,
-                                            timeout=self._param.timeout)
+                if method == "get":
+                    response = requests.get(url=url, params=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
                     if self._param.clean_html:
                         sections = HtmlParser()(None, response.content)
                         self.set_output("result", "\n".join(sections))
                     else:
                         self.set_output("result", response.text)
 
-                if method == 'put':
-                    if self._param.datatype.lower() == 'json':
-                        response = requests.put(url=url,
-                                                json=args,
-                                                headers=headers,
-                                                proxies=proxies,
-                                                timeout=self._param.timeout)
+                if method == "put":
+                    if self._param.datatype.lower() == "json":
+                        response = requests.put(url=url, json=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
                     else:
-                        response = requests.put(url=url,
-                                                data=args,
-                                                headers=headers,
-                                                proxies=proxies,
-                                                timeout=self._param.timeout)
+                        response = requests.put(url=url, data=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
                     if self._param.clean_html:
                         sections = HtmlParser()(None, response.content)
                         self.set_output("result", "\n".join(sections))
                     else:
                         self.set_output("result", response.text)
 
-                if method == 'post':
-                    if self._param.datatype.lower() == 'json':
-                        response = requests.post(url=url,
-                                                 json=args,
-                                                 headers=headers,
-                                                 proxies=proxies,
-                                                 timeout=self._param.timeout)
+                if method == "post":
+                    if self._param.datatype.lower() == "json":
+                        response = requests.post(url=url, json=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
                     else:
-                        response = requests.post(url=url,
-                                                 data=args,
-                                                 headers=headers,
-                                                 proxies=proxies,
-                                                 timeout=self._param.timeout)
+                        response = requests.post(url=url, data=args, headers=headers, proxies=proxies, timeout=self._param.timeout)
                     if self._param.clean_html:
                         self.set_output("result", "\n".join(sections))
                     else:
@@ -128,6 +127,9 @@ class Invoke(ComponentBase, ABC):
 
                 return self.output("result")
             except Exception as e:
+                if self.check_if_canceled("Invoke processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"Http request error: {e}")
                 time.sleep(self._param.delay_after_error)

agent/component/iteration.py

@@ -16,6 +16,13 @@
 from abc import ABC
 from agent.component.base import ComponentBase, ComponentParamBase
 
+"""
+class VariableModel(BaseModel):
+    data_type: Annotated[Literal["string", "number", "Object", "Boolean", "Array<string>", "Array<number>", "Array<object>", "Array<boolean>"], Field(default="Array<string>")]
+    input_mode: Annotated[Literal["constant", "variable"], Field(default="constant")]
+    value: Annotated[Any, Field(default=None)]
+    model_config = ConfigDict(extra="forbid")
+"""
 
 class IterationParam(ComponentParamBase):
     """
@@ -49,6 +56,9 @@ class Iteration(ComponentBase, ABC):
                 return cid
 
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Iteration processing"):
+            return
+
         arr = self._canvas.get_variable_value(self._param.items_ref)
         if not isinstance(arr, list):
             self.set_output("_ERROR", self._param.items_ref + " must be an array, but its type is "+str(type(arr)))

agent/component/iterationitem.py

@@ -33,6 +33,9 @@ class IterationItem(ComponentBase, ABC):
         self._idx = 0
 
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("IterationItem processing"):
+            return
+
         parent = self.get_parent()
         arr = self._canvas.get_variable_value(parent._param.items_ref)
         if not isinstance(arr, list):
@@ -40,12 +43,17 @@ class IterationItem(ComponentBase, ABC):
             raise Exception(parent._param.items_ref + " must be an array, but its type is "+str(type(arr)))
 
         if self._idx > 0:
+            if self.check_if_canceled("IterationItem processing"):
+                return
             self.output_collation()
 
         if self._idx >= len(arr):
             self._idx = -1
             return
 
+        if self.check_if_canceled("IterationItem processing"):
+            return
+
         self.set_output("item", arr[self._idx])
         self.set_output("index", self._idx)
 
@@ -80,4 +88,4 @@ class IterationItem(ComponentBase, ABC):
         return self._idx == -1
 
     def thoughts(self) -> str:
-        return "Next turn..."
+        return "Next turn..."

agent/component/llm.py

@@ -21,13 +21,12 @@ from copy import deepcopy
 from typing import Any, Generator
 import json_repair
 from functools import partial
-from api.db import LLMType
+from common.constants import LLMType
 from api.db.services.llm_service import LLMBundle
 from api.db.services.tenant_llm_service import TenantLLMService
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.api_utils import timeout
-from rag.prompts import message_fit_in, citation_prompt
-from rag.prompts.prompts import tool_call_summary
+from common.connection_utils import timeout
+from rag.prompts.generator import tool_call_summary, message_fit_in, citation_prompt, structured_output_prompt
 
 
 class LLMParam(ComponentParamBase):
@@ -82,9 +81,9 @@ class LLMParam(ComponentParamBase):
 
 class LLM(ComponentBase):
     component_name = "LLM"
-    
-    def __init__(self, canvas, id, param: ComponentParamBase):
-        super().__init__(canvas, id, param)
+
+    def __init__(self, canvas, component_id, param: ComponentParamBase):
+        super().__init__(canvas, component_id, param)
         self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), TenantLLMService.llm_id2llm_type(self._param.llm_id),
                                   self._param.llm_id, max_retries=self._param.max_retries,
                                   retry_interval=self._param.delay_after_error
@@ -102,6 +101,8 @@ class LLM(ComponentBase):
 
     def get_input_elements(self) -> dict[str, Any]:
         res = self.get_input_elements_from_text(self._param.sys_prompt)
+        if isinstance(self._param.prompts, str):
+            self._param.prompts = [{"role": "user", "content": self._param.prompts}]
         for prompt in self._param.prompts:
             d = self.get_input_elements_from_text(prompt["content"])
             res.update(d)
@@ -113,6 +114,17 @@ class LLM(ComponentBase):
     def add2system_prompt(self, txt):
         self._param.sys_prompt += txt
 
+    def _sys_prompt_and_msg(self, msg, args):
+        if isinstance(self._param.prompts, str):
+            self._param.prompts = [{"role": "user", "content": self._param.prompts}]
+        for p in self._param.prompts:
+            if msg and msg[-1]["role"] == p["role"]:
+                continue
+            p = deepcopy(p)
+            p["content"] = self.string_format(p["content"], args)
+            msg.append(p)
+        return msg, self.string_format(self._param.sys_prompt, args)
+
     def _prepare_prompt_variables(self):
         if self._param.visual_files_var:
             self.imgs = self._canvas.get_variable_value(self._param.visual_files_var)
@@ -128,7 +140,6 @@ class LLM(ComponentBase):
 
         args = {}
         vars = self.get_input_elements() if not self._param.debug_inputs else self._param.debug_inputs
-        sys_prompt = self._param.sys_prompt
         for k, o in vars.items():
             args[k] = o["value"]
             if not isinstance(args[k], str):
@@ -138,16 +149,8 @@ class LLM(ComponentBase):
                     args[k] = str(args[k])
             self.set_input_value(k, args[k])
 
-        msg = self._canvas.get_history(self._param.message_history_window_size)[:-1]
-        for p in self._param.prompts:
-            if msg and msg[-1]["role"] == p["role"]:
-                continue
-            msg.append(deepcopy(p))
-
-        sys_prompt = self.string_format(sys_prompt, args)
+        msg, sys_prompt = self._sys_prompt_and_msg(self._canvas.get_history(self._param.message_history_window_size)[:-1], args)
         user_defined_prompt, sys_prompt = self._extract_prompts(sys_prompt)
-        for m in msg:
-            m["content"] = self.string_format(m["content"], args)
         if self._param.cite and self._canvas.get_reference()["chunks"]:
             sys_prompt += citation_prompt(user_defined_prompt)
 
@@ -202,20 +205,30 @@ class LLM(ComponentBase):
             for txt in self.chat_mdl.chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs):
                 yield delta(txt)
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("LLM processing"):
+            return
+
         def clean_formated_answer(ans: str) -> str:
             ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
             ans = re.sub(r"^.*```json", "", ans, flags=re.DOTALL)
             return re.sub(r"```\n*$", "", ans, flags=re.DOTALL)
 
         prompt, msg, _ = self._prepare_prompt_variables()
-        error = ""
-
-        if self._param.output_structure:
-            prompt += "\nThe output MUST follow this JSON format:\n"+json.dumps(self._param.output_structure, ensure_ascii=False, indent=2)
-            prompt += "\nRedundant information is FORBIDDEN."
+        error: str = ""
+        output_structure=None
+        try:
+            output_structure = self._param.outputs['structured']
+        except Exception:
+            pass
+        if output_structure:
+            schema=json.dumps(output_structure, ensure_ascii=False, indent=2)
+            prompt += structured_output_prompt(schema)
             for _ in range(self._param.max_retries+1):
+                if self.check_if_canceled("LLM processing"):
+                    return
+
                 _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97))
                 error = ""
                 ans = self._generate(msg)
@@ -225,7 +238,7 @@ class LLM(ComponentBase):
                     error = ans
                     continue
                 try:
-                    self.set_output("structured_content", json_repair.loads(clean_formated_answer(ans)))
+                    self.set_output("structured", json_repair.loads(clean_formated_answer(ans)))
                     return
                 except Exception:
                     msg.append({"role": "user", "content": "The answer can't not be parsed as JSON"})
@@ -236,11 +249,14 @@ class LLM(ComponentBase):
 
         downstreams = self._canvas.get_component(self._id)["downstream"] if self._canvas.get_component(self._id) else []
         ex = self.exception_handler()
-        if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not self._param.output_structure and not (ex and ex["goto"]):
+        if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not output_structure and not (ex and ex["goto"]):
             self.set_output("content", partial(self._stream_output, prompt, msg))
             return
 
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("LLM processing"):
+                return
+
             _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97))
             error = ""
             ans = self._generate(msg)
@@ -262,6 +278,9 @@ class LLM(ComponentBase):
         _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97))
         answer = ""
         for ans in self._generate_streamly(msg):
+            if self.check_if_canceled("LLM streaming"):
+                return
+
             if ans.find("**ERROR**") >= 0:
                 if self.get_exception_default_value():
                     self.set_output("content", self.get_exception_default_value())
@@ -280,4 +299,4 @@ class LLM(ComponentBase):
 
     def thoughts(self) -> str:
         _, msg,_ = self._prepare_prompt_variables()
-        return "⌛Give me a moment—starting from: \n\n" + re.sub(r"(User's query:|[\\]+)", '', msg[-1]['content'], flags=re.DOTALL) + "\n\nI’ll figure out our best next move."
+        return "⌛Give me a moment—starting from: \n\n" + re.sub(r"(User's query:|[\\]+)", '', msg[-1]['content'], flags=re.DOTALL) + "\n\nI’ll figure out our best next move."

agent/component/message.py

@@ -23,7 +23,7 @@ from typing import Any
 from agent.component.base import ComponentBase, ComponentParamBase
 from jinja2 import Template as Jinja2Template
 
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class MessageParam(ComponentParamBase):
@@ -49,7 +49,10 @@ class MessageParam(ComponentParamBase):
 class Message(ComponentBase):
     component_name = "Message"
 
-    def get_kwargs(self, script:str, kwargs:dict = {}, delimeter:str=None) -> tuple[str, dict[str, str | list | Any]]:
+    def get_input_elements(self) -> dict[str, Any]:
+        return self.get_input_elements_from_text("".join(self._param.content))
+
+    def get_kwargs(self, script:str, kwargs:dict = {}, delimiter:str=None) -> tuple[str, dict[str, str | list | Any]]:
         for k,v in self.get_input_elements_from_text(script).items():
             if k in kwargs:
                 continue
@@ -60,8 +63,8 @@ class Message(ComponentBase):
             if isinstance(v, partial):
                 for t in v():
                     ans += t
-            elif isinstance(v, list) and delimeter:
-                ans = delimeter.join([str(vv) for vv in v])
+            elif isinstance(v, list) and delimiter:
+                ans = delimiter.join([str(vv) for vv in v])
             elif not isinstance(v, str):
                 try:
                     ans = json.dumps(v, ensure_ascii=False)
@@ -86,6 +89,9 @@ class Message(ComponentBase):
         all_content = ""
         cache = {}
         for r in re.finditer(self.variable_ref_patt, rand_cnt, flags=re.DOTALL):
+            if self.check_if_canceled("Message streaming"):
+                return
+
             all_content += rand_cnt[s: r.start()]
             yield rand_cnt[s: r.start()]
             s = r.end()
@@ -96,26 +102,33 @@ class Message(ComponentBase):
                 continue
 
             v = self._canvas.get_variable_value(exp)
-            if not v:
+            if v is None:
                 v = ""
             if isinstance(v, partial):
                 cnt = ""
                 for t in v():
+                    if self.check_if_canceled("Message streaming"):
+                        return
+
                     all_content += t
                     cnt += t
                     yield t
-
+                self.set_input_value(exp, cnt)
                 continue
             elif not isinstance(v, str):
                 try:
-                    v = json.dumps(v, ensure_ascii=False, indent=2)
+                    v = json.dumps(v, ensure_ascii=False)
                 except Exception:
                     v = str(v)
             yield v
+            self.set_input_value(exp, v)
             all_content += v
             cache[exp] = v
 
         if s < len(rand_cnt):
+            if self.check_if_canceled("Message streaming"):
+                return
+
             all_content += rand_cnt[s: ]
             yield rand_cnt[s: ]
 
@@ -127,8 +140,11 @@ class Message(ComponentBase):
         ]
         return any([re.search(p, content) for p in patt])
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Message processing"):
+            return
+
         rand_cnt = random.choice(self._param.content)
         if self._param.stream and not self._is_jinjia2(rand_cnt):
             self.set_output("content", partial(self._stream, rand_cnt))
@@ -141,6 +157,9 @@ class Message(ComponentBase):
         except Exception:
             pass
 
+        if self.check_if_canceled("Message processing"):
+            return
+
         for n, v in kwargs.items():
             content = re.sub(n, v, content)
 

agent/component/string_transform.py

@@ -16,9 +16,11 @@
 import os
 import re
 from abc import ABC
+from typing import Any
+
 from jinja2 import Template as Jinja2Template
 from agent.component.base import ComponentParamBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 from .message import Message
 
 
@@ -43,6 +45,9 @@ class StringTransformParam(ComponentParamBase):
 class StringTransform(Message, ABC):
     component_name = "StringTransform"
 
+    def get_input_elements(self) -> dict[str, Any]:
+        return self.get_input_elements_from_text(self._param.script)
+
     def get_input_form(self) -> dict[str, dict]:
         if self._param.method == "split":
             return {
@@ -56,19 +61,26 @@ class StringTransform(Message, ABC):
             "type": "line"
         } for k, o in self.get_input_elements_from_text(self._param.script).items()}
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("StringTransform processing"):
+            return
+
         if self._param.method == "split":
             self._split(kwargs.get("line"))
         else:
             self._merge(kwargs)
 
     def _split(self, line:str|None = None):
+        if self.check_if_canceled("StringTransform split processing"):
+            return
+
         var = self._canvas.get_variable_value(self._param.split_ref) if not line else line
         if not var:
             var = ""
         assert isinstance(var, str), "The input variable is not a string: {}".format(type(var))
         self.set_input_value(self._param.split_ref, var)
+
         res = []
         for i,s in enumerate(re.split(r"(%s)"%("|".join([re.escape(d) for d in self._param.delimiters])), var, flags=re.DOTALL)):
             if i % 2 == 1:
@@ -77,6 +89,9 @@ class StringTransform(Message, ABC):
         self.set_output("result", res)
 
     def _merge(self, kwargs:dict[str, str] = {}):
+        if self.check_if_canceled("StringTransform merge processing"):
+            return
+
         script = self._param.script
         script, kwargs = self.get_kwargs(script, kwargs, self._param.delimiters[0])
 
@@ -90,7 +105,7 @@ class StringTransform(Message, ABC):
         for k,v in kwargs.items():
             if not v:
                 v = ""
-            script = re.sub(k, v, script)
+            script = re.sub(k, lambda match: v, script)
 
         self.set_output("result", script)
 

agent/component/switch.py

@@ -19,7 +19,7 @@ from abc import ABC
 from typing import Any
 
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class SwitchParam(ComponentParamBase):
@@ -61,11 +61,20 @@ class SwitchParam(ComponentParamBase):
 class Switch(ComponentBase, ABC):
     component_name = "Switch"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 3))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 3)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Switch processing"):
+            return
+
         for cond in self._param.conditions:
+            if self.check_if_canceled("Switch processing"):
+                return
+
             res = []
             for item in cond["items"]:
+                if self.check_if_canceled("Switch processing"):
+                    return
+
                 if not item["cpn_id"]:
                     continue
                 cpn_v = self._canvas.get_variable_value(item["cpn_id"])
@@ -128,4 +137,4 @@ class Switch(ComponentBase, ABC):
         raise ValueError('Not supported operator' + operator)
 
     def thoughts(self) -> str:
-        return "I’m weighing a few options and will pick the next step shortly."
+        return "I’m weighing a few options and will pick the next step shortly."

agent/component/varaiable_aggregator.py

@@ -0,0 +1,84 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import Any
+import os
+
+from common.connection_utils import timeout
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class VariableAggregatorParam(ComponentParamBase):
+    """
+    Parameters for VariableAggregator
+
+    - groups: list of dicts {"group_name": str, "variables": [variable selectors]}
+    """
+
+    def __init__(self):
+        super().__init__()
+        # each group expects: {"group_name": str, "variables": List[str]}
+        self.groups = []
+
+    def check(self):
+        self.check_empty(self.groups, "[VariableAggregator] groups")
+        for g in self.groups:
+            if not g.get("group_name"):
+                raise ValueError("[VariableAggregator] group_name can not be empty!")
+            if not g.get("variables"):
+                raise ValueError(
+                    f"[VariableAggregator] variables of group `{g.get('group_name')}` can not be empty"
+                )
+            if not isinstance(g.get("variables"), list):
+                raise ValueError(
+                    f"[VariableAggregator] variables of group `{g.get('group_name')}` should be a list of strings"
+                )
+
+    def get_input_form(self) -> dict[str, dict]:
+        return {
+            "variables": {
+                "name": "Variables",
+                "type": "list",
+            }
+        }
+
+
+class VariableAggregator(ComponentBase):
+    component_name = "VariableAggregator"
+
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 3)))
+    def _invoke(self, **kwargs):
+        # Group mode: for each group, pick the first available variable
+        for group in self._param.groups:
+            gname = group.get("group_name")
+
+            # record candidate selectors within this group
+            self.set_input_value(f"{gname}.variables", list(group.get("variables", [])))
+            for selector in group.get("variables", []):
+                val = self._canvas.get_variable_value(selector['value'])
+                if val:
+                    self.set_output(gname, val)
+                    break
+            
+    @staticmethod
+    def _to_object(value: Any) -> Any:
+        # Try to convert value to serializable object if it has to_object()
+        try:
+            return value.to_object()  # type: ignore[attr-defined]
+        except Exception:
+            return value
+
+    def thoughts(self) -> str:
+        return "Aggregating variables from canvas and grouping as configured."

agent/component/webhook.py

@@ -0,0 +1,38 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from agent.component.base import ComponentParamBase, ComponentBase
+
+
+class WebhookParam(ComponentParamBase):
+
+    """
+    Define the Begin component parameters.
+    """
+    def __init__(self):
+        super().__init__()
+
+    def get_input_form(self) -> dict[str, dict]:
+        return getattr(self, "inputs")
+
+
+class Webhook(ComponentBase):
+    component_name = "Webhook"
+
+    def _invoke(self, **kwargs):
+        pass
+
+    def thoughts(self) -> str:
+        return ""

agent/templates/generate_SEO_blog.json

@@ -2,9 +2,11 @@
     "id": 8,
     "title": {
         "en": "Generate SEO Blog",
+        "de": "SEO Blog generieren",
         "zh": "生成SEO博客"},
     "description": {
         "en": "This is a multi-agent version of the SEO blog generation workflow. It simulates a small team of AI “writers”, where each agent plays a specialized role — just like a real editorial team.",
+        "de": "Dies ist eine Multi-Agenten-Version des Workflows zur Erstellung von SEO-Blogs. Sie simuliert ein kleines Team von KI-„Autoren“, in dem jeder Agent eine spezielle Rolle übernimmt – genau wie in einem echten Redaktionsteam.",
         "zh": "多智能体架构可根据简单的用户输入自动生成完整的SEO博客文章。模拟小型“作家”团队，其中每个智能体扮演一个专业角色——就像真正的编辑团队。"},
     "canvas_type": "Agent",
     "dsl": {

agent/templates/knowledge_base_report.json

@@ -2,9 +2,11 @@
     "id": 20,
     "title": {
         "en": "Report Agent Using Knowledge Base",
+        "de": "Berichtsagent mit Wissensdatenbank",
         "zh": "知识库检索智能体"},
     "description": {
         "en": "A report generation assistant using local knowledge base, with advanced capabilities in task planning, reasoning, and reflective analysis. Recommended for academic research paper Q&A",
+        "de": "Ein Berichtsgenerierungsassistent, der eine lokale Wissensdatenbank nutzt, mit erweiterten Fähigkeiten in Aufgabenplanung, Schlussfolgerung und reflektierender Analyse. Empfohlen für akademische Forschungspapier-Fragen und -Antworten.",
         "zh": "一个使用本地知识库的报告生成助手，具备高级能力，包括任务规划、推理和反思性分析。推荐用于学术研究论文问答。"},
     "canvas_type": "Agent",
     "dsl": {

agent/templates/knowledge_base_report_r.json

@@ -1,10 +1,12 @@
 {
     "id": 21,
     "title": {
-        "en": "Report Agent Using Knowledge Base", 
+        "en": "Report Agent Using Knowledge Base",
+        "de": "Berichtsagent mit Wissensdatenbank",
         "zh": "知识库检索智能体"},
     "description": {
         "en": "A report generation assistant using local knowledge base, with advanced capabilities in task planning, reasoning, and reflective analysis. Recommended for academic research paper Q&A",
+        "de": "Ein Berichtsgenerierungsassistent, der eine lokale Wissensdatenbank nutzt, mit erweiterten Fähigkeiten in Aufgabenplanung, Schlussfolgerung und reflektierender Analyse. Empfohlen für akademische Forschungspapier-Fragen und -Antworten.",
         "zh": "一个使用本地知识库的报告生成助手，具备高级能力，包括任务规划、推理和反思性分析。推荐用于学术研究论文问答。"},
     "canvas_type": "Recommended",
     "dsl": {

agent/templates/market_generate_seo_blog.json

@@ -2,9 +2,11 @@
     "id": 12,
     "title": {
         "en": "Generate SEO Blog",
+        "de": "SEO Blog generieren",
         "zh": "生成SEO博客"},
     "description": {
-        "en": "This workflow automatically generates a complete SEO-optimized blog article based on a simple user input. You don’t need any writing experience. Just provide a topic or short request — the system will handle the rest.",
+        "en": "This workflow automatically generates a complete SEO-optimized blog article based on a simple user input. You don't need any writing experience. Just provide a topic or short request — the system will handle the rest.",
+        "de": "Dieser Workflow generiert automatisch einen vollständigen SEO-optimierten Blogartikel basierend auf einer einfachen Benutzereingabe. Sie benötigen keine Schreiberfahrung. Geben Sie einfach ein Thema oder eine kurze Anfrage ein – das System übernimmt den Rest.",
         "zh": "此工作流根据简单的用户输入自动生成完整的SEO博客文章。你无需任何写作经验，只需提供一个主题或简短请求，系统将处理其余部分。"},
     "canvas_type": "Marketing",
     "dsl": {
@@ -916,4 +918,4 @@
             "retrieval": []
         },
     "avatar": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAwADADASIAAhEBAxEB/8QAGQAAAwEBAQAAAAAAAAAAAAAABgkKBwUI/8QAMBAAAAYCAQIEBQQCAwAAAAAAAQIDBAUGBxEhCAkAEjFBFFFhcaETFiKRFyOx8PH/xAAaAQACAwEBAAAAAAAAAAAAAAACAwABBgQF/8QALBEAAgIBAgUCBAcAAAAAAAAAAQIDBBEFEgATITFRIkEGIzJhFBUWgaGx8P/aAAwDAQACEQMRAD8AfF2hez9089t7pvxgQMa1Gb6qZ6oQE9m/NEvCIStyPfJSOF/M1epzMugo/qtMqbiRc1mJjoJKCLMNIxKcsLJedfO1Ct9cI63x9fx6CA/19t+oh4LFA5HfuAgP/A8eOIsnsTBrkBHXA7+v53+Q+ficTgJft9gIgA+/P9/1r342O/YA8A8k3/if+IbAN7+2/f8AAiI6H19PGoPyESTMZQPKUAHkQEN+3r9dh78/YPGUTk2wb/qAZZIugH1OHH5DjkdfbnWw2DsOxPj+xjrnx2H39unBopJGBn9s+PHv1HXjPJtH+J+B40O9a16h/wB/92j/ALrPa/wR104UyAobHlXhuo2HrEtK4qy3CwjKOuJLRHJLSkXWrFKs/gVrJVrE8TUiH8bPrP20UEu8m4hNpMJJuTOfnbUw/kUqyZgMHGjAO9+mtDsQ53sdcB6eMhnpEjhNQxRKICAgHy5+/roOdjr7c+J6O4x07dx484/n7nzw1gexBGfIPkZ/3t39uGpqc6+fP5/Ht8vGFZCzJjWpWuBxvO2yPjrtclUUK7BqmUI4fuASeyhG5FzFI0Bw4aQ0iZNoDgzvRW4qtyFkI4XmwyEk2YNnDp0sVBu3IUyy5iqH8gqKERSIRNIii67hddRJs1at01Xbx2sgzZoLu10UFJR+4V1A5cxF3FqNcLvjwcno43uuLrOxZYjujaClcb4QQfxEizpFiQyM9olcueRnjC2ZMt9iY06zL0qytrMSqSOVGsfHMaGhZ3l4lSRI2MqE74zJvRTveNFWWIh3RWw+XCAM5icKQLrCH57T17FhErSlRXnWvyZXKQwWJ3eraD14p5YuZCFgacskK2oGkVuKO5GYTHzf7DaD12cBD3DgPOIDrWw9PnrXPgDkpVsUDGMG+DD6E9gHXIjrYjwUPQTCXYgHPhIV974+F6E1hpC14Yzmzj56YaQEeZhXsayD1zLPW7pygxaMf81Nzu1iJsnIuDIKnaJAkPldqrHaoORZ73tMVEbFdSXT9nVgRQgnBq6j8e/HCIEATpAnH5KlmRVkFRFJwks/bqImSXJ5VFyA3N6Ikh3bCW3YHp5cowOmCfTgA+xJCnrjtwHKcLvJj2ZGcTRFj19kEhckdzgEjKnABGSSzdc1Fe5byXXGNjKdvRcw5NxvLidNZFFCxUa62KrzMaChw8hhYScFJtROAgmuLByq1MsgkZYPaVVuDe0wraRaqAdJwgRQo+YR8xTlAQNx6b49w41vXiJpCalLh1jZhyrTqRM4+jstdRmYryNkydLQRWg1LNGcWd5jIFFvCythlIySa0mNu74sKRQtaWsTmupqPItw0lE52ufpyYzrSkx6cw5bLmBEpkTsz+dt8P5QFuCRtAIkBH9MuwKHICIaDQhnojMs9mKaeGcrMxXlQtAYkdVljimRrE5MqI4zL8oSqQ6wxjodBqK05qdK3Vo3aCSVkBW7bjuC1NFJJBPaqyx6fp6pWkliYLXK2XrukkRu2CCVoSWMgsdMyySKwoLFcIGWSTUMg4IBgTcICoBhRcplMcpFkhIqQp1ClMBTmA0Zfe1zpjvHfXff65bZlzXpB3jjGTgiirmPjAfs16PHqHeQ75Wbj3xxZpOEkV3LRJJSPdomUBZISJLncV2k+8D07dxXp7xsYuTapA9UkJUYWIzNhadnWEZeCXGLQQiJi1ViHfhHL2unWh+mlORsrW0JFpEFnGVfm1mU4kq0FY3eD6corJncv6dr5NLSMNXVaTUksjTiMnaq8uFfSVuDyiJ1iZpy0LOJtpa3YfkcQ5fdozyxI2m5qqcrHN61YYmHsh6v3o9ParYmYJEtlhIx6+gUbjgD23M6oqg92YL0JyF6Bps+qDValVA9h9Lj5SZI3SHXdEQlj1wiQtLLIe6pGzjO3BlBkK1hxpblLVH5wdW0BcFKf/JwRtjsot2z8omaSdxbzzk1iEjsE0AM9rrRZNRIrVyo7dGO6E+oh8axLlJ5H5VaJKx7ePRGFbW6vUeFfHQIWPTI9Tm7HHfuhqY7E6C7JFqUzM6iZXIoncNxX7+bIVdJnTT48x3OQU1krIDW3UeixVhyISzYz6cadY5Xph6TseRNTRsTElzzBn9Vlly0TAERsdgnMYyLROjyFbg5R4ZlsGaMT4yNi2Zlq1GwjZB3jq0PsaJfA3t0jL0W0Y9xf1V41lpWckXMLaZiwxuKYPqc6LlHdkeRF+Qxswx5ASDqBVrsL+2A/N6SiCbYymV2BywJiMZj3GRRMTnL+lVyHCll3R7Szv0vqXMtQ74T+HijljIScLaEpkKCB3rqMBIi0jPs5JeOKTZMZEi5VVnouzy0k3jXjWSMlY6UcVGDxlKMVDqx91SILWSi3D2KdgYy3kP8E9X/AE1SnRXBNdNRMlefT6g7aY6giK+cPLGNg0bY68rcnpsNh9PqIBve/EcPQ3WIq2dR93xpSgk5SAZ9R6MLAOZFUkpLSUDXp6/KPpGUkmTdswlnKnwbl5ITMdGwcXJi7LKsqzUmT5tWYmkXuF9wjBvb76b7dHheazJ9RElUJOCxViuMlUJC0Gtz6PKyjLBY4qMWUe12r1xZ6lOyT6XPEBKN2CkTDOlZd02TBdTMt7Upx2knrkdCv1UKjDKn1A7XBYH6SCOOrWn5Oi/DtRiu+GleRthDL8rXdVjZlcfWrSIxVlGGGCOnH//Z"
-}
+}

agent/templates/seo_blog.json

@@ -2,9 +2,11 @@
     "id": 4,
     "title": {
         "en": "Generate SEO Blog",
+        "de": "SEO Blog generieren",
         "zh": "生成SEO博客"},
     "description": {
-        "en": "This workflow automatically generates a complete SEO-optimized blog article based on a simple user input. You don’t need any writing experience. Just provide a topic or short request — the system will handle the rest.",
+        "en": "This workflow automatically generates a complete SEO-optimized blog article based on a simple user input. You don't need any writing experience. Just provide a topic or short request — the system will handle the rest.",
+        "de": "Dieser Workflow generiert automatisch einen vollständigen SEO-optimierten Blogartikel basierend auf einer einfachen Benutzereingabe. Sie benötigen keine Schreiberfahrung. Geben Sie einfach ein Thema oder eine kurze Anfrage ein – das System übernimmt den Rest.",
         "zh": "此工作流根据简单的用户输入自动生成完整的SEO博客文章。你无需任何写作经验，只需提供一个主题或简短请求，系统将处理其余部分。"},
     "canvas_type": "Recommended",
     "dsl": {
@@ -916,4 +918,4 @@
             "retrieval": []
         },
     "avatar": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAwADADASIAAhEBAxEB/8QAGQAAAwEBAQAAAAAAAAAAAAAABgkKBwUI/8QAMBAAAAYCAQIEBQQCAwAAAAAAAQIDBAUGBxEhCAkAEjFBFFFhcaETFiKRFyOx8PH/xAAaAQACAwEBAAAAAAAAAAAAAAACAwABBgQF/8QALBEAAgIBAgUCBAcAAAAAAAAAAQIDBBEFEgATITFRIkEGIzJhFBUWgaGx8P/aAAwDAQACEQMRAD8AfF2hez9089t7pvxgQMa1Gb6qZ6oQE9m/NEvCIStyPfJSOF/M1epzMugo/qtMqbiRc1mJjoJKCLMNIxKcsLJedfO1Ct9cI63x9fx6CA/19t+oh4LFA5HfuAgP/A8eOIsnsTBrkBHXA7+v53+Q+ficTgJft9gIgA+/P9/1r342O/YA8A8k3/if+IbAN7+2/f8AAiI6H19PGoPyESTMZQPKUAHkQEN+3r9dh78/YPGUTk2wb/qAZZIugH1OHH5DjkdfbnWw2DsOxPj+xjrnx2H39unBopJGBn9s+PHv1HXjPJtH+J+B40O9a16h/wB/92j/ALrPa/wR104UyAobHlXhuo2HrEtK4qy3CwjKOuJLRHJLSkXWrFKs/gVrJVrE8TUiH8bPrP20UEu8m4hNpMJJuTOfnbUw/kUqyZgMHGjAO9+mtDsQ53sdcB6eMhnpEjhNQxRKICAgHy5+/roOdjr7c+J6O4x07dx484/n7nzw1gexBGfIPkZ/3t39uGpqc6+fP5/Ht8vGFZCzJjWpWuBxvO2yPjrtclUUK7BqmUI4fuASeyhG5FzFI0Bw4aQ0iZNoDgzvRW4qtyFkI4XmwyEk2YNnDp0sVBu3IUyy5iqH8gqKERSIRNIii67hddRJs1at01Xbx2sgzZoLu10UFJR+4V1A5cxF3FqNcLvjwcno43uuLrOxZYjujaClcb4QQfxEizpFiQyM9olcueRnjC2ZMt9iY06zL0qytrMSqSOVGsfHMaGhZ3l4lSRI2MqE74zJvRTveNFWWIh3RWw+XCAM5icKQLrCH57T17FhErSlRXnWvyZXKQwWJ3eraD14p5YuZCFgacskK2oGkVuKO5GYTHzf7DaD12cBD3DgPOIDrWw9PnrXPgDkpVsUDGMG+DD6E9gHXIjrYjwUPQTCXYgHPhIV974+F6E1hpC14Yzmzj56YaQEeZhXsayD1zLPW7pygxaMf81Nzu1iJsnIuDIKnaJAkPldqrHaoORZ73tMVEbFdSXT9nVgRQgnBq6j8e/HCIEATpAnH5KlmRVkFRFJwks/bqImSXJ5VFyA3N6Ikh3bCW3YHp5cowOmCfTgA+xJCnrjtwHKcLvJj2ZGcTRFj19kEhckdzgEjKnABGSSzdc1Fe5byXXGNjKdvRcw5NxvLidNZFFCxUa62KrzMaChw8hhYScFJtROAgmuLByq1MsgkZYPaVVuDe0wraRaqAdJwgRQo+YR8xTlAQNx6b49w41vXiJpCalLh1jZhyrTqRM4+jstdRmYryNkydLQRWg1LNGcWd5jIFFvCythlIySa0mNu74sKRQtaWsTmupqPItw0lE52ufpyYzrSkx6cw5bLmBEpkTsz+dt8P5QFuCRtAIkBH9MuwKHICIaDQhnojMs9mKaeGcrMxXlQtAYkdVljimRrE5MqI4zL8oSqQ6wxjodBqK05qdK3Vo3aCSVkBW7bjuC1NFJJBPaqyx6fp6pWkliYLXK2XrukkRu2CCVoSWMgsdMyySKwoLFcIGWSTUMg4IBgTcICoBhRcplMcpFkhIqQp1ClMBTmA0Zfe1zpjvHfXff65bZlzXpB3jjGTgiirmPjAfs16PHqHeQ75Wbj3xxZpOEkV3LRJJSPdomUBZISJLncV2k+8D07dxXp7xsYuTapA9UkJUYWIzNhadnWEZeCXGLQQiJi1ViHfhHL2unWh+mlORsrW0JFpEFnGVfm1mU4kq0FY3eD6corJncv6dr5NLSMNXVaTUksjTiMnaq8uFfSVuDyiJ1iZpy0LOJtpa3YfkcQ5fdozyxI2m5qqcrHN61YYmHsh6v3o9ParYmYJEtlhIx6+gUbjgD23M6oqg92YL0JyF6Bps+qDValVA9h9Lj5SZI3SHXdEQlj1wiQtLLIe6pGzjO3BlBkK1hxpblLVH5wdW0BcFKf/JwRtjsot2z8omaSdxbzzk1iEjsE0AM9rrRZNRIrVyo7dGO6E+oh8axLlJ5H5VaJKx7ePRGFbW6vUeFfHQIWPTI9Tm7HHfuhqY7E6C7JFqUzM6iZXIoncNxX7+bIVdJnTT48x3OQU1krIDW3UeixVhyISzYz6cadY5Xph6TseRNTRsTElzzBn9Vlly0TAERsdgnMYyLROjyFbg5R4ZlsGaMT4yNi2Zlq1GwjZB3jq0PsaJfA3t0jL0W0Y9xf1V41lpWckXMLaZiwxuKYPqc6LlHdkeRF+Qxswx5ASDqBVrsL+2A/N6SiCbYymV2BywJiMZj3GRRMTnL+lVyHCll3R7Szv0vqXMtQ74T+HijljIScLaEpkKCB3rqMBIi0jPs5JeOKTZMZEi5VVnouzy0k3jXjWSMlY6UcVGDxlKMVDqx91SILWSi3D2KdgYy3kP8E9X/AE1SnRXBNdNRMlefT6g7aY6giK+cPLGNg0bY68rcnpsNh9PqIBve/EcPQ3WIq2dR93xpSgk5SAZ9R6MLAOZFUkpLSUDXp6/KPpGUkmTdswlnKnwbl5ITMdGwcXJi7LKsqzUmT5tWYmkXuF9wjBvb76b7dHheazJ9RElUJOCxViuMlUJC0Gtz6PKyjLBY4qMWUe12r1xZ6lOyT6XPEBKN2CkTDOlZd02TBdTMt7Upx2knrkdCv1UKjDKn1A7XBYH6SCOOrWn5Oi/DtRiu+GleRthDL8rXdVjZlcfWrSIxVlGGGCOnH//Z"
-}
+}

agent/templates/sql_assistant.json

@@ -2,10 +2,12 @@
     "id": 17,
     "title": {
         "en": "SQL Assistant",
+        "de": "SQL Assistent",
         "zh": "SQL助理"},
     "description": {
-        "en": "SQL Assistant is an AI-powered tool that lets business users turn plain-English questions into fully formed SQL queries. Simply type your question (e.g., “Show me last quarter’s top 10 products by revenue”) and SQL Assistant generates the exact SQL, runs it against your database, and returns the results in seconds. ",
-        "zh": "用户能够将简单文本问题转化为完整的SQL查询并输出结果。只需输入您的问题（例如，“展示上个季度前十名按收入排序的产品”），SQL助理就会生成精确的SQL语句，对其运行您的数据库，并几秒钟内返回结果。"},
+        "en": "SQL Assistant is an AI-powered tool that lets business users turn plain-English questions into fully formed SQL queries. Simply type your question (e.g., 'Show me last quarter's top 10 products by revenue') and SQL Assistant generates the exact SQL, runs it against your database, and returns the results in seconds. ",
+        "de": "SQL-Assistent ist ein KI-gestütztes Tool, mit dem Geschäftsanwender einfache englische Fragen in vollständige SQL-Abfragen umwandeln können. Geben Sie einfach Ihre Frage ein (z.B. 'Zeige mir die Top 10 Produkte des letzten Quartals nach Umsatz') und der SQL-Assistent generiert das exakte SQL, führt es gegen Ihre Datenbank aus und liefert die Ergebnisse in Sekunden.",
+        "zh": "用户能够将简单文本问题转化为完整的SQL查询并输出结果。只需输入您的问题（例如，展示上个季度前十名按收入排序的产品），SQL助理就会生成精确的SQL语句，对其运行您的数据库，并几秒钟内返回结果。"},
     "canvas_type": "Marketing",
     "dsl": {
             "components": {
@@ -83,7 +85,7 @@
                             },
                             "password": "20010812Yy!",
                             "port": 3306,
-                            "sql": "Agent:WickedGoatsDivide@content",
+                            "sql": "{Agent:WickedGoatsDivide@content}",
                             "username": "13637682833@163.com"
                         }
                     },
@@ -114,9 +116,7 @@
                         "params": {
                             "cross_languages": [],
                             "empty_response": "",
-                            "kb_ids": [
-                                "ed31364c727211f0bdb2bafe6e7908e6"
-                            ],
+                            "kb_ids": [],
                             "keywords_similarity_weight": 0.7,
                             "outputs": {
                                 "formalized_content": {
@@ -124,7 +124,7 @@
                                     "value": ""
                                 }
                             },
-                            "query": "sys.query",
+                            "query": "{sys.query}",
                             "rerank_id": "",
                             "similarity_threshold": 0.2,
                             "top_k": 1024,
@@ -145,9 +145,7 @@
                         "params": {
                             "cross_languages": [],
                             "empty_response": "",
-                            "kb_ids": [
-                                "0f968106727311f08357bafe6e7908e6"
-                            ],
+                            "kb_ids": [],
                             "keywords_similarity_weight": 0.7,
                             "outputs": {
                                 "formalized_content": {
@@ -155,7 +153,7 @@
                                     "value": ""
                                 }
                             },
-                            "query": "sys.query",
+                            "query": "{sys.query}",
                             "rerank_id": "",
                             "similarity_threshold": 0.2,
                             "top_k": 1024,
@@ -176,9 +174,7 @@
                         "params": {
                             "cross_languages": [],
                             "empty_response": "",
-                            "kb_ids": [
-                                "4ad1f9d0727311f0827dbafe6e7908e6"
-                            ],
+                            "kb_ids": [],
                             "keywords_similarity_weight": 0.7,
                             "outputs": {
                                 "formalized_content": {
@@ -186,7 +182,7 @@
                                     "value": ""
                                 }
                             },
-                            "query": "sys.query",
+                            "query": "{sys.query}",
                             "rerank_id": "",
                             "similarity_threshold": 0.2,
                             "top_k": 1024,
@@ -347,9 +343,7 @@
                             "form": {
                                 "cross_languages": [],
                                 "empty_response": "",
-                                "kb_ids": [
-                                    "ed31364c727211f0bdb2bafe6e7908e6"
-                                ],
+                                "kb_ids": [],
                                 "keywords_similarity_weight": 0.7,
                                 "outputs": {
                                     "formalized_content": {
@@ -357,7 +351,7 @@
                                         "value": ""
                                     }
                                 },
-                                "query": "sys.query",
+                                "query": "{sys.query}",
                                 "rerank_id": "",
                                 "similarity_threshold": 0.2,
                                 "top_k": 1024,
@@ -387,9 +381,7 @@
                             "form": {
                                 "cross_languages": [],
                                 "empty_response": "",
-                                "kb_ids": [
-                                    "0f968106727311f08357bafe6e7908e6"
-                                ],
+                                "kb_ids": [],
                                 "keywords_similarity_weight": 0.7,
                                 "outputs": {
                                     "formalized_content": {
@@ -397,7 +389,7 @@
                                         "value": ""
                                     }
                                 },
-                                "query": "sys.query",
+                                "query": "{sys.query}",
                                 "rerank_id": "",
                                 "similarity_threshold": 0.2,
                                 "top_k": 1024,
@@ -427,9 +419,7 @@
                             "form": {
                                 "cross_languages": [],
                                 "empty_response": "",
-                                "kb_ids": [
-                                    "4ad1f9d0727311f0827dbafe6e7908e6"
-                                ],
+                                "kb_ids": [],
                                 "keywords_similarity_weight": 0.7,
                                 "outputs": {
                                     "formalized_content": {
@@ -437,7 +427,7 @@
                                         "value": ""
                                     }
                                 },
-                                "query": "sys.query",
+                                "query": "{sys.query}",
                                 "rerank_id": "",
                                 "similarity_threshold": 0.2,
                                 "top_k": 1024,
@@ -539,7 +529,7 @@
                                 },
                                 "password": "20010812Yy!",
                                 "port": 3306,
-                                "sql": "Agent:WickedGoatsDivide@content",
+                                "sql": "{Agent:WickedGoatsDivide@content}",
                                 "username": "13637682833@163.com"
                             },
                             "label": "ExeSQL",
@@ -725,4 +715,4 @@
             "retrieval": []
         },
     "avatar": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAcFBQYFBAcGBQYIBwcIChELCgkJChUPEAwRGBUaGRgVGBcbHichGx0lHRcYIi4iJSgpKywrGiAvMy8qMicqKyr/2wBDAQcICAoJChQLCxQqHBgcKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKir/wAARCAAwADADAREAAhEBAxEB/8QAGgAAAwEBAQEAAAAAAAAAAAAABQYHBAMAAf/EADIQAAEDAwMCBAMHBQAAAAAAAAECAwQFESEABjESEyJBUYEUYXEHFSNSkaGxMjNictH/xAAZAQADAQEBAAAAAAAAAAAAAAACAwQBAAX/xAAlEQACAgICAgEEAwAAAAAAAAABAgARAyESMQRBEyIycYFCkbH/2gAMAwEAAhEDEQA/AKHt2DGpNHXDLrZdWtSrIub39tZ5GbGwPA+pmDFkX7x7idvra85xqQaFNkxUTVIVJQzf8QpBFjbgEenNs681MnA9WJ6fEOKJoxVpSpFLTCo6KEZlTlLcQBIJS20hAv1D1ve+qPk52b0IsYuIGtyt7ZkVVNP+H3A5GdlN2u7GQUBSfmkk8cXH10tmLD6Yl0CG5qmTXBMZiQEMuvupUoKdc6UeEi4FsqOeBxrsKnv1AY+hJ2l5yfu6qQ6/UZtPDRHZ+Eldpsqz1hSrXJGLXwRxqxUQizFs7galPYUFDKT+h15oMuImspQpFiL+2i1A3A1bgxmixUgwlT8ZfgJ/y8P8HXdRuPZoxaqtfkQKbKqF03jtEoDeFKV1lNgfK4H764XfccVUgipvdiwKpFaXMLklFg4juuqV0m3Izg/MaEZCDYMScYqiJOd6xmqfUVfBJcWwtHV1Elfi87k51ViyhrsxL4ivQj1KrFZjTGjTJ8aShdyph5SUqFhwPzX9jpC0dXUqZK3ViHNq7oNaVJjz2Vw5LCrdKknpULZyfMf801MfI1e5NmpAGHUL12EZNFWWlhXSUuWHKgk3xomwEDuDhzLysySU9EndEVyIz3GmxJR+KpBIdCLlRHn/AFEjjIF9AMJlZ8gLZ/qUiJSg1Tu0HO4plFj4FC1h9NYfHIU7kwzgnqCJlKLiCO2s6hKytWiPJoFdfnLW7HS0or6bqXbjg2AI99XjAa3NPlL6jFTduOR5sd1+oyfjQMONqI7QOMA4V7/pqjHjC9SLNn56I1HiqrqTUKM0hbq2lpst5CQSST54xjSPJbICOHUhawISiRQ02T2Uq6AAkqFj/GquJQks1iEr/INLU82bploKSFXusG9xfjHofXQuQUNRoQqQT0ZwVEST5687iZWGgpDsebNbaTDfKVL/ALnbQU/UkKNhjXpFt0BJBVXe/wAGGG6YMlvvNkjlBGmKeJimHIVc0TY89akCKspT28C5BKgDyR7fvrCFI+q/1DQsvVfudYcVyKw49KU6tZyQbmwHFhrOKr9s0uz0CAIpbr3RKo1Rbh02C4HJISp2ZIz0pJ8IQk5Nr/QXznSX6NSnGAwHI/gD/TM+3vtAj1arJpcpgtPdPSH0kFt5wDxAWOOLgamIAFwijCfD927N2tGXuNxlK2W0occUhJWpR+QzzrPjc+pvyqT3Ftf2zbObf7YYecb6CrrDAGfy20wYMkA5Vjbtev7b3nEcXRela27d1ogoWi/rnQsjrqZzHdwzKoKUsqWz3mOnJUlZJt8uokD621w+RdzgynUkUpoUafPZXMnSHlrKluyX1Eug8XF7GwxbgWxrubMO5WmNRsCKtLfcY3rAU0nIltkBP+w0X8Jjdz//2Q=="
-}
+}

agent/templates/user_interaction.json

@@ -0,0 +1,519 @@
+{
+    "id": 27,
+    "title": {
+        "en": "Interactive Agent",
+        "zh": "可交互的 Agent"
+    },
+    "description": {
+        "en": "During the Agent’s execution, users can actively intervene and interact with the Agent to adjust or guide its output, ensuring the final result aligns with their intentions.",
+        "zh": "在 Agent 的运行过程中，用户可以随时介入，与 Agent 进行交互，以调整或引导生成结果，使最终输出更符合预期。"
+    },
+    "canvas_type": "Agent",
+    "dsl": {
+        "components": {
+            "Agent:LargeFliesMelt": {
+            "downstream": [
+                "UserFillUp:GoldBroomsRelate"
+            ],
+            "obj": {
+                "component_name": "Agent",
+                "params": {
+                "cite": true,
+                "delay_after_error": 1,
+                "description": "",
+                "exception_default_value": "",
+                "exception_goto": [],
+                "exception_method": "",
+                "frequencyPenaltyEnabled": false,
+                "frequency_penalty": 0.7,
+                "llm_id": "qwen-turbo@Tongyi-Qianwen",
+                "maxTokensEnabled": false,
+                "max_retries": 3,
+                "max_rounds": 1,
+                "max_tokens": 256,
+                "mcp": [],
+                "message_history_window_size": 12,
+                "outputs": {
+                    "content": {
+                    "type": "string",
+                    "value": ""
+                    },
+                    "structured": {}
+                },
+                "presencePenaltyEnabled": false,
+                "presence_penalty": 0.4,
+                "prompts": [
+                    {
+                    "content": "User query:{sys.query}",
+                    "role": "user"
+                    }
+                ],
+                "sys_prompt": "<role>\nYou are the Planning Agent in a multi-agent RAG workflow.\nYour sole job is to design a crisp, executable Search Plan for the next agent. Do not search or answer the user’s question.\n</role>\n<objectives>\nUnderstand the user’s task and decompose it into evidence-seeking steps.\nProduce high-quality queries and retrieval settings tailored to the task type (fact lookup, multi-hop reasoning, comparison, statistics, how-to, etc.).\nIdentify missing information that would materially change the plan (≤3 concise questions).\nOptimize for source trustworthiness, diversity, and recency; define stopping criteria to avoid over-searching.\nAnswer in 150 words.\n<objectives>",
+                "temperature": 0.1,
+                "temperatureEnabled": false,
+                "tools": [],
+                "topPEnabled": false,
+                "top_p": 0.3,
+                "user_prompt": "",
+                "visual_files_var": ""
+                }
+            },
+            "upstream": [
+                "begin"
+            ]
+            },
+            "Agent:TangyWordsType": {
+            "downstream": [
+                "Message:FreshWallsStudy"
+            ],
+            "obj": {
+                "component_name": "Agent",
+                "params": {
+                "cite": true,
+                "delay_after_error": 1,
+                "description": "",
+                "exception_default_value": "",
+                "exception_goto": [],
+                "exception_method": "",
+                "frequencyPenaltyEnabled": false,
+                "frequency_penalty": 0.7,
+                "llm_id": "qwen-turbo@Tongyi-Qianwen",
+                "maxTokensEnabled": false,
+                "max_retries": 3,
+                "max_rounds": 1,
+                "max_tokens": 256,
+                "mcp": [],
+                "message_history_window_size": 12,
+                "outputs": {
+                    "content": {
+                    "type": "string",
+                    "value": ""
+                    },
+                    "structured": {}
+                },
+                "presencePenaltyEnabled": false,
+                "presence_penalty": 0.4,
+                "prompts": [
+                    {
+                    "content": "Search Plan: {Agent:LargeFliesMelt@content}\n\n\n\nAwait Response feedback:{UserFillUp:GoldBroomsRelate@instructions}\n",
+                    "role": "user"
+                    }
+                ],
+                "sys_prompt": "<role>\nYou are the Search Agent.\nYour job is to execute the approved Search Plan, integrate the Await Response feedback, retrieve evidence, and produce a well-grounded answer.\n</role>\n<objectives>\nTranslate the plan + feedback into concrete searches.\nCollect diverse, trustworthy, and recent evidence meeting the plan’s evidence bar.\nSynthesize a concise answer; include citations next to claims they support.\nIf evidence is insufficient or conflicting, clearly state limitations and propose next steps.\n</objectives>\n <tools>\nRetrieval: You must use Retrieval to do the search.\n </tools>\n",
+                "temperature": 0.1,
+                "temperatureEnabled": false,
+                "tools": [
+                    {
+                    "component_name": "Retrieval",
+                    "name": "Retrieval",
+                    "params": {
+                        "cross_languages": [],
+                        "description": "",
+                        "empty_response": "",
+                        "kb_ids": [],
+                        "keywords_similarity_weight": 0.7,
+                        "outputs": {
+                        "formalized_content": {
+                            "type": "string",
+                            "value": ""
+                        },
+                        "json": {
+                            "type": "Array<Object>",
+                            "value": []
+                        }
+                        },
+                        "rerank_id": "",
+                        "similarity_threshold": 0.2,
+                        "toc_enhance": false,
+                        "top_k": 1024,
+                        "top_n": 8,
+                        "use_kg": false
+                    }
+                    }
+                ],
+                "topPEnabled": false,
+                "top_p": 0.3,
+                "user_prompt": "",
+                "visual_files_var": ""
+                }
+            },
+            "upstream": [
+                "UserFillUp:GoldBroomsRelate"
+            ]
+            },
+            "Message:FreshWallsStudy": {
+            "downstream": [],
+            "obj": {
+                "component_name": "Message",
+                "params": {
+                "content": [
+                    "{Agent:TangyWordsType@content}"
+                ]
+                }
+            },
+            "upstream": [
+                "Agent:TangyWordsType"
+            ]
+            },
+            "UserFillUp:GoldBroomsRelate": {
+            "downstream": [
+                "Agent:TangyWordsType"
+            ],
+            "obj": {
+                "component_name": "UserFillUp",
+                "params": {
+                "enable_tips": true,
+                "inputs": {
+                    "instructions": {
+                    "name": "instructions",
+                    "optional": false,
+                    "options": [],
+                    "type": "paragraph"
+                    }
+                },
+                "outputs": {
+                    "instructions": {
+                    "name": "instructions",
+                    "optional": false,
+                    "options": [],
+                    "type": "paragraph"
+                    }
+                },
+                "tips": "Here is my search plan:\n{Agent:LargeFliesMelt@content}\nAre you okay with it?"
+                }
+            },
+            "upstream": [
+                "Agent:LargeFliesMelt"
+            ]
+            },
+            "begin": {
+            "downstream": [
+                "Agent:LargeFliesMelt"
+            ],
+            "obj": {
+                "component_name": "Begin",
+                "params": {}
+            },
+            "upstream": []
+            }
+        },
+        "globals": {
+            "sys.conversation_turns": 0,
+            "sys.files": [],
+            "sys.query": "",
+            "sys.user_id": ""
+        },
+        "graph": {
+            "edges": [
+            {
+                "data": {
+                "isHovered": false
+                },
+                "id": "xy-edge__beginstart-Agent:LargeFliesMeltend",
+                "source": "begin",
+                "sourceHandle": "start",
+                "target": "Agent:LargeFliesMelt",
+                "targetHandle": "end"
+            },
+            {
+                "data": {
+                "isHovered": false
+                },
+                "id": "xy-edge__Agent:LargeFliesMeltstart-UserFillUp:GoldBroomsRelateend",
+                "source": "Agent:LargeFliesMelt",
+                "sourceHandle": "start",
+                "target": "UserFillUp:GoldBroomsRelate",
+                "targetHandle": "end"
+            },
+            {
+                "data": {
+                "isHovered": false
+                },
+                "id": "xy-edge__UserFillUp:GoldBroomsRelatestart-Agent:TangyWordsTypeend",
+                "source": "UserFillUp:GoldBroomsRelate",
+                "sourceHandle": "start",
+                "target": "Agent:TangyWordsType",
+                "targetHandle": "end"
+            },
+            {
+                "id": "xy-edge__Agent:TangyWordsTypetool-Tool:NastyBatsGoend",
+                "source": "Agent:TangyWordsType",
+                "sourceHandle": "tool",
+                "target": "Tool:NastyBatsGo",
+                "targetHandle": "end"
+            },
+            {
+                "id": "xy-edge__Agent:TangyWordsTypestart-Message:FreshWallsStudyend",
+                "source": "Agent:TangyWordsType",
+                "sourceHandle": "start",
+                "target": "Message:FreshWallsStudy",
+                "targetHandle": "end"
+            }
+            ],
+            "nodes": [
+            {
+                "data": {
+                "label": "Begin",
+                "name": "begin"
+                },
+                "dragging": false,
+                "id": "begin",
+                "measured": {
+                "height": 50,
+                "width": 200
+                },
+                "position": {
+                "x": 154.9008789064451,
+                "y": 119.51001744285344
+                },
+                "selected": false,
+                "sourcePosition": "left",
+                "targetPosition": "right",
+                "type": "beginNode"
+            },
+            {
+                "data": {
+                "form": {
+                    "cite": true,
+                    "delay_after_error": 1,
+                    "description": "",
+                    "exception_default_value": "",
+                    "exception_goto": [],
+                    "exception_method": "",
+                    "frequencyPenaltyEnabled": false,
+                    "frequency_penalty": 0.7,
+                    "llm_id": "qwen-turbo@Tongyi-Qianwen",
+                    "maxTokensEnabled": false,
+                    "max_retries": 3,
+                    "max_rounds": 1,
+                    "max_tokens": 256,
+                    "mcp": [],
+                    "message_history_window_size": 12,
+                    "outputs": {
+                    "content": {
+                        "type": "string",
+                        "value": ""
+                    },
+                    "structured": {}
+                    },
+                    "presencePenaltyEnabled": false,
+                    "presence_penalty": 0.4,
+                    "prompts": [
+                    {
+                        "content": "User query:{sys.query}",
+                        "role": "user"
+                    }
+                    ],
+                    "sys_prompt": "<role>\nYou are the Planning Agent in a multi-agent RAG workflow.\nYour sole job is to design a crisp, executable Search Plan for the next agent. Do not search or answer the user’s question.\n</role>\n<objectives>\nUnderstand the user’s task and decompose it into evidence-seeking steps.\nProduce high-quality queries and retrieval settings tailored to the task type (fact lookup, multi-hop reasoning, comparison, statistics, how-to, etc.).\nIdentify missing information that would materially change the plan (≤3 concise questions).\nOptimize for source trustworthiness, diversity, and recency; define stopping criteria to avoid over-searching.\nAnswer in 150 words.\n<objectives>",
+                    "temperature": 0.1,
+                    "temperatureEnabled": false,
+                    "tools": [],
+                    "topPEnabled": false,
+                    "top_p": 0.3,
+                    "user_prompt": "",
+                    "visual_files_var": ""
+                },
+                "label": "Agent",
+                "name": "Planning Agent"
+                },
+                "dragging": false,
+                "id": "Agent:LargeFliesMelt",
+                "measured": {
+                "height": 90,
+                "width": 200
+                },
+                "position": {
+                "x": 443.96309330796714,
+                "y": 104.61370811205677
+                },
+                "selected": false,
+                "sourcePosition": "right",
+                "targetPosition": "left",
+                "type": "agentNode"
+            },
+            {
+                "data": {
+                "form": {
+                    "enable_tips": true,
+                    "inputs": {
+                    "instructions": {
+                        "name": "instructions",
+                        "optional": false,
+                        "options": [],
+                        "type": "paragraph"
+                    }
+                    },
+                    "outputs": {
+                    "instructions": {
+                        "name": "instructions",
+                        "optional": false,
+                        "options": [],
+                        "type": "paragraph"
+                    }
+                    },
+                    "tips": "Here is my search plan:\n{Agent:LargeFliesMelt@content}\nAre you okay with it?"
+                },
+                "label": "UserFillUp",
+                "name": "Await Response"
+                },
+                "dragging": false,
+                "id": "UserFillUp:GoldBroomsRelate",
+                "measured": {
+                "height": 50,
+                "width": 200
+                },
+                "position": {
+                "x": 683.3409492927474,
+                "y": 116.76274137645598
+                },
+                "selected": false,
+                "sourcePosition": "right",
+                "targetPosition": "left",
+                "type": "ragNode"
+            },
+            {
+                "data": {
+                "form": {
+                    "cite": true,
+                    "delay_after_error": 1,
+                    "description": "",
+                    "exception_default_value": "",
+                    "exception_goto": [],
+                    "exception_method": "",
+                    "frequencyPenaltyEnabled": false,
+                    "frequency_penalty": 0.7,
+                    "llm_id": "qwen-turbo@Tongyi-Qianwen",
+                    "maxTokensEnabled": false,
+                    "max_retries": 3,
+                    "max_rounds": 1,
+                    "max_tokens": 256,
+                    "mcp": [],
+                    "message_history_window_size": 12,
+                    "outputs": {
+                    "content": {
+                        "type": "string",
+                        "value": ""
+                    },
+                    "structured": {}
+                    },
+                    "presencePenaltyEnabled": false,
+                    "presence_penalty": 0.4,
+                    "prompts": [
+                    {
+                        "content": "Search Plan: {Agent:LargeFliesMelt@content}\n\n\n\nAwait Response feedback:{UserFillUp:GoldBroomsRelate@instructions}\n",
+                        "role": "user"
+                    }
+                    ],
+                    "sys_prompt": "<role>\nYou are the Search Agent.\nYour job is to execute the approved Search Plan, integrate the Await Response feedback, retrieve evidence, and produce a well-grounded answer.\n</role>\n<objectives>\nTranslate the plan + feedback into concrete searches.\nCollect diverse, trustworthy, and recent evidence meeting the plan’s evidence bar.\nSynthesize a concise answer; include citations next to claims they support.\nIf evidence is insufficient or conflicting, clearly state limitations and propose next steps.\n</objectives>\n <tools>\nRetrieval: You must use Retrieval to do the search.\n </tools>\n",
+                    "temperature": 0.1,
+                    "temperatureEnabled": false,
+                    "tools": [
+                    {
+                        "component_name": "Retrieval",
+                        "name": "Retrieval",
+                        "params": {
+                        "cross_languages": [],
+                        "description": "",
+                        "empty_response": "",
+                        "kb_ids": [],
+                        "keywords_similarity_weight": 0.7,
+                        "outputs": {
+                            "formalized_content": {
+                            "type": "string",
+                            "value": ""
+                            },
+                            "json": {
+                            "type": "Array<Object>",
+                            "value": []
+                            }
+                        },
+                        "rerank_id": "",
+                        "similarity_threshold": 0.2,
+                        "toc_enhance": false,
+                        "top_k": 1024,
+                        "top_n": 8,
+                        "use_kg": false
+                        }
+                    }
+                    ],
+                    "topPEnabled": false,
+                    "top_p": 0.3,
+                    "user_prompt": "",
+                    "visual_files_var": ""
+                },
+                "label": "Agent",
+                "name": "Search Agent"
+                },
+                "dragging": false,
+                "id": "Agent:TangyWordsType",
+                "measured": {
+                "height": 90,
+                "width": 200
+                },
+                "position": {
+                "x": 944.6411255659472,
+                "y": 99.84499066368488
+                },
+                "selected": true,
+                "sourcePosition": "right",
+                "targetPosition": "left",
+                "type": "agentNode"
+            },
+            {
+                "data": {
+                "form": {
+                    "description": "This is an agent for a specific task.",
+                    "user_prompt": "This is the order you need to send to the agent."
+                },
+                "label": "Tool",
+                "name": "flow.tool_0"
+                },
+                "id": "Tool:NastyBatsGo",
+                "measured": {
+                "height": 50,
+                "width": 200
+                },
+                "position": {
+                "x": 862.6411255659472,
+                "y": 239.84499066368488
+                },
+                "sourcePosition": "right",
+                "targetPosition": "left",
+                "type": "toolNode"
+            },
+            {
+                "data": {
+                "form": {
+                    "content": [
+                    "{Agent:TangyWordsType@content}"
+                    ]
+                },
+                "label": "Message",
+                "name": "Message"
+                },
+                "dragging": false,
+                "id": "Message:FreshWallsStudy",
+                "measured": {
+                "height": 50,
+                "width": 200
+                },
+                "position": {
+                "x": 1216.7057997987163,
+                "y": 120.48541298149814
+                },
+                "selected": false,
+                "sourcePosition": "right",
+                "targetPosition": "left",
+                "type": "messageNode"
+            }
+            ]
+        },
+        "history": [],
+        "messages": [],
+        "path": [],
+        "retrieval": [],
+        "variables": {}
+        },
+        "avatar":
+    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAACXBIWXMAABYlAAAWJQFJUiTwAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAA1FSURBVHgBzVppcFRVFv7e672TTjrprGTrkI1tgFjiAI4TmgEGcEFgVFAHxr2mLFe0cKxyBOeHljVjwVDqzDhVTqksOlOiuCBrCwk7hCAkLAGy70kn3eklvb03571e8rrTCQGi5em6ecu9fe85557znXNuh0EUHTGb5/Ast4TnmXsB3oifBTFVxEsVy7PrZ5lM9RE9oZvTZrPeAe51Gvh85ABGMuonJh6Ra9Mzw2KDm2PXm0ymPoS6ReZ5n5n6p4f45QOsj0ihEXxwdGg9Pta3Ax2R98ErH+wIzMFjFFTlhcwkCMEKT4qE+Nc5np8e5Aa8MAf94UNtmEl5DPYFrpHPER8+xn3U3KNkXqDpKtFaSGi1Wm188MH7655+6inoExLR0toW0N211R+8jJ198YHtiFyDH5EHE8Oy7Iccx/1BeJ4yZRI2bXgHSfokyGQsLH19YDgax4RXGJz450A8v1HGMMw62soMEgRxcVqcrDyG9X95E339fSidPh05Odnot9vB0Qe84CB8wMQglSe8HWPI3HAmJVmLYTOYJ55YxVedOYcTxyux59svsfqpJ5GWkorGhhZYLL0oKSnC2pfX4DemOaIg9n473B63ZCFGOu2YU4QIgikIJiF5yaxcuZTPzzfC42WQnZWJLdu2IiXFgG+/3oOs7Ey0t3fC7/cjMyMdU6dOwpyyMswtmwO9PhmdnYE+5hrcS9Eq8MzSHRfuvT7Dj5qbiKeGY4fKsWL1ShQWFKG1uQVz55Thh5pqVJ87D6fLDY/HA7/PD87PiYxMnToRLzz7LI0zoamphQTxYewpKIzUB6OUJcIoOTEOVpjhcroRH69FQ2MLHANOZKZn4PcPP4DikgKoVWrI5CzkcjmUSiUuXriKRx77I2bMuh0tba1IStKL84TXDTU26hp9z0QzJYVVbhBuJdDL8Vy4yeRK+TrBNHp7e9HR0QWXawDNTa3o7ulB2R2zER+nQXZ2FhRKBVwOl7BloO+IVwGpBtxebPv0v0hNN+CO2beHHT4cD6I9Pvo+iobElmtYk6xkQuE6nvOjpaWNmHeDJxPp7bVCp4vHvv0HsXfvQVSeOgOtRoOFi+aJu+C0O6FVa2AwJEOhkMPhcODEiVNwuJ0w5ubRLsbB7/UPMsvH5HTYxvNBwSUAIT5yQcG4wTlk3+/5bt2cX5chIzMdbW3tOH/+EtRaNex2B1RkKkuXLkJCog41NZdw8MBhFBUVICU1WTQDwZwMKcloJJN74snHMeByoq7uCr1XECBkEzB4I3ge6p4xkp3hiOFj+rdcYFpg5vaZs3H3nXehh0znyPFjWLv2NcyeNQPbtn0hDlRr1GKc2L1rP5QqJeQyMiMZRH8QqKujDa+9+gruWf4A4siPlLQzeXlG+Dl/hAjR2B7KnURQEnyDC4kyXPoS5dN7dn7FR0umJ4d8aNUjBKEdePSxh8iMDuDixcsiCglbK5PJ4PMFUEe4F6B01zfbMeD1iTx8vPUTMi0lXn7uJXT1dIr+Eph/EFViAMqoKGBag89sZFIVuMaTplsJWZYtvwv7yQdqai5CQWYhoMyECUWYv8BE2s0RJ8gk03v/3Q24UHsJ27/YjnaKDa4BD1INBpQfrgDlWoFZRRPgcdNBW4hlLBNGM3mEdMGrmzDfSNt/4OBhdLR1Yv26tXj77U0iEqUYUvDvf34AjUpOaUclssaNQ2cXoZdzAC++/Cds3vwZLl+oxtZPt2INPVefOYH6hsbALtwkDabtg8jG8pK0OURtbR34dMtHaKhrImlZvPXW35FsSMKKFfeivOIwsnKMWLDwbrjdbtEHOB8HFfnF8YoD+Md7m7Bv3y7s2rtfXOjkqUqMFYnMR5tQsCdCQwK+t5MQ+3d/g4cfXiHCZHeXBZ98/D/R5gX0sfRZsGTZAzDNW4SG5kbk5uSSL3DIzc1GaekMkNyQ0Z+PN29FYmICxooEPkNN5Hv3zh0jhgq9PhFXrtSh+nw1Kg4dIah04Oix02JfQUEepk2bgs8//0bcocWLfotXXlqD7u5uzF1wp7hIcnIyKr7fjSZKTyDV5HXkO9EkrfjksQZI62Brnw2pqQbMz5yH5fcug4/zUtCqxIcffSRmsI2NrSKjlp5ebNnyGWG/B2++8QYW3TlfTAhZGQOej8EAw0SYbcTaGLk6kxZR8miGY4wWGRAY6+7pFu1PSLH/88G/yAcG8NcNG3H85CkxSgrBT4gPDqcDE4sn4GjKCbBCsGAki7KDO8AybIBRugqLMFFMhuPDCALJI9CBGfxidKkYfi+Mp0mFGCEkUy888wy0Wq3YJyf/sFOa0W/rR093DzLS0tDXZw3WHkPnE9li5IhzdMCu0EGupnn8/sgxLEISBJTMISKSsVKnCDEtvYY+ovLIKQVGQk1waCF56yDs7+zsotjRDlu/DR6fF4sXL0RN9QWwFD/ik1JJwdwQ7XEspSKudqSqFUhsqcWAtUccH4tC/Ilrh/hlmbB84RcjtZgTSwQMPQtayhqXiTPvvIDHJyXjtWVlUCUkQ6ZQBHZDXIvMR6akOmMALtoxTUom9K2XaPesYl9wsmsSG2aQYTDih4ndQvl89LOb0o72wlkonTYZ9y25BwffWYOexstQJ+jhJ8377b1Ic9ngam4A53GC9zqgpCrPd7YcnFwZqB4l64aVHNXYmBoMDkbMgmMUJNqqH6rEFChvW0wMWjFv0RKom37AuX3bkcnbUKCOQ3LqOFgb69Gu1GGgs5HiiwYTbp0JS8UOMBpdxLrDRXI2WruICmrRGh5VEz9kIj4PtIZMqGdQ1HZZEZ+ZhxmFuVD73WT/bhx5/8+wZk1AvHEqrAYjCdoHH9UUxRMnoefIDggFHitjR9QVK9VyQIBYzNxAHhPEeI6EkJM2vfm3QSnzwpA/FZaOeuzdvAn9GUYUlC2Gj2zfkJoPRfpkNFdW0DgWeVlGKJqr4SKQkFNmOxwHbDSTTFjxTFAuaQ9z7Y8EIUK+RdgIXq1DUsFM2Pva0dfWDGu/C8ZxBhiaTyGXbF/ltsBSfRI+dQKhmB8KbTwUcYlgG2vQ095CZaxy0KwxaOLscEyEtyWYx4ey4ZhloFQ/XLD5o5pgDl46V+q4igOU6PVyGtxy10ooEtPQfKYc+z/cBBfrRM7EKZR3dYs7x/IK6JJT4K2vRq8QTxgpfwgaaizn4MN/pFsyfAsWKRFNFlCPCNRC3k7PjVZKCpvrULZ8BV7927vobmxG7dG9OHK4HDYqW6/W1cI3wCAtdyql8c0U/Z3wUuGUWzwFF8w7oIzTiZYptQ2WCUdoVnKVoFL4lgn3D+5UYKx45YVdCupEsmNhUyRUUhrywGWUoGhCKdpbWmGpO42LZ8/BR/EgnmqNxb97FCqFH9UHd8JG6YjT1U9w7IOP6pNJhUYc3f0VtCSElBt5KDeJFEKKnbwElQLhX0oMI70O52rBBM3nRELRL9HQeh5x8KGy3IzjVbWwWfvxwc7dsFssuHz2NBo7W1Ciywdnt4KOQ+ChkxC1SoWmc+XwmBZRYaUWq0Oe80H26KpV60TNhTQW0fiAVoNFBBPSMs9E+ED08/CNxgm2HZcCy5XjkGt16Dp/Fms3vUfiyGFtv4KK/ftQWFxEhwEMvFQwyTihWFKIJ4Ie+u6V0+XkUjzcDhvcNJ9ckkxgSN6LoNfzsYA08t3ooTYA1UrjrXD+cAC/mj+X0CYJ/cT8oe++ppQiBf2OfjE5tFD2qzekIk9GdXpGDtRUSMnpN4wByptSUhOhdLVK6gFecu4SfY3Nxo0TbX18qhHujHqkjy9G84WT6GptwZUOC2qqziM7Lx16jRI54/MJwTzgKYdiBSgWYopSQ0edJXB6/NhyslYCo+F8Q5KDYGiUHVTiTYkAn9MGfcks9NFRpnAUqc/MIgY5LF1xH9Ioe1247H5MnjQNxZNvAUMptttuQyYdlrXRaYmccqnaLhvSEjWRFdlozEDK+I0KIVZiAsRSkaSdOBfeumPobWkUS9RcOqZJjtchJbsEjt42eEhQpS5ZFBiETHVX68jnPHR040YGHW1GFjQxXED67mbq2IhpJTvJu21QjZ8BFaXRPnq2kP3rE3RouUBVHsGzlQ4PEhLi4CB/aCLo9dLhmRDFs+k3jEt0GC3sQD01Y2jCoatJb6MGXEOe0QnMUPCyI+OWBdDk/AK1x3ejraFeLJYG6DTE7XIhjiC0z9IFJ9UNOq2AOz4UZ2bD6hyokj22enU+zTETN0LMyO16MliOzEmpVsH4i1lIzy2kn0+14CjAUSYHi80uFj8cOfL4wiJkpKfT4QKHLB37HVNuNs9hWc6MkVU5IiL9GCSccMsVKjrOl5HZuKnWtqKj4TI6as+AcffTkaUKfl6WL9rEoQP7NpAenhvNxEOO92LGjx+BGCGfUkKh0ZIv21Bz9vTGJ59+8XlRALPZrFexvLAL0zEGNNLJ80h910FVmoQkU2lpaeBfDYT/OXBzjIm0uhFjQMwN9o2KOH6jhngVmI853xGz2QgZ1lH2MI3yIXFHeP4nNP7YVE9cfMlw7BezTKbvpR3/Bx465XnKBextAAAAAElFTkSuQmCC"
+}

agent/test/client.py

@@ -16,7 +16,7 @@
 import argparse
 import os
 from agent.canvas import Canvas
-from api import settings
+from common import settings
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

agent/tools/arxiv.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 import arxiv
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class ArXivParam(ToolParamBase):
@@ -61,14 +61,20 @@ class ArXivParam(ToolParamBase):
 class ArXiv(ToolBase, ABC):
     component_name = "ArXiv"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("ArXiv processing"):
+            return
+
         if not kwargs.get("query"):
             self.set_output("formalized_content", "")
             return ""
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("ArXiv processing"):
+                return
+
             try:
                 sort_choices = {"relevance": arxiv.SortCriterion.Relevance,
                                 "lastUpdatedDate": arxiv.SortCriterion.LastUpdatedDate,
@@ -79,12 +85,20 @@ class ArXiv(ToolBase, ABC):
                     max_results=self._param.top_n,
                     sort_by=sort_choices[self._param.sort_by]
                 )
-                self._retrieve_chunks(list(arxiv_client.results(search)),
+                results = list(arxiv_client.results(search))
+
+                if self.check_if_canceled("ArXiv processing"):
+                    return
+
+                self._retrieve_chunks(results,
                                       get_title=lambda r: r.title,
                                       get_url=lambda r: r.pdf_url,
                                       get_content=lambda r: r.summary)
                 return self.output("formalized_content")
             except Exception as e:
+                if self.check_if_canceled("ArXiv processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"ArXiv error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -97,6 +111,6 @@ class ArXiv(ToolBase, ABC):
 
     def thoughts(self) -> str:
         return """
-Keywords: {} 
+Keywords: {}
 Looking for the most relevant articles.
-                """.format(self.get_input().get("query", "-_-!"))
+                """.format(self.get_input().get("query", "-_-!"))

agent/tools/base.py

@@ -20,9 +20,9 @@ from copy import deepcopy
 from functools import partial
 from typing import TypedDict, List, Any
 from agent.component.base import ComponentParamBase, ComponentBase
-from api.utils import hash_str2int
+from common.misc_utils import hash_str2int
 from rag.llm.chat_model import ToolCallSession
-from rag.prompts.prompts import kb_prompt
+from rag.prompts.generator import kb_prompt
 from rag.utils.mcp_tool_call_conn import MCPToolCallSession
 from timeit import default_timer as timer
 
@@ -125,6 +125,9 @@ class ToolBase(ComponentBase):
         return self._param.get_meta()
 
     def invoke(self, **kwargs):
+        if self.check_if_canceled("Tool processing"):
+            return
+
         self.set_output("_created_time", time.perf_counter())
         try:
             res = self._invoke(**kwargs)
@@ -170,4 +173,4 @@ class ToolBase(ComponentBase):
         self.set_output("formalized_content", "\n".join(kb_prompt({"chunks": chunks, "doc_aggs": aggs}, 200000, True)))
 
     def thoughts(self) -> str:
-        return self._canvas.get_component_name(self._id) + " is running..."
+        return self._canvas.get_component_name(self._id) + " is running..."

agent/tools/code_exec.py

@@ -21,8 +21,8 @@ from strenum import StrEnum
 from typing import Optional
 from pydantic import BaseModel, Field, field_validator
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api import settings
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
+from common import settings
 
 
 class Language(StrEnum):
@@ -129,12 +129,16 @@ module.exports = { main };
 class CodeExec(ToolBase, ABC):
     component_name = "CodeExec"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("CodeExec processing"):
+            return
+
         lang = kwargs.get("lang", self._param.lang)
         script = kwargs.get("script", self._param.script)
         arguments = {}
         for k, v in self._param.arguments.items():
+
             if kwargs.get(k):
                 arguments[k] = kwargs[k]
                 continue
@@ -149,15 +153,28 @@ class CodeExec(ToolBase, ABC):
     def _execute_code(self, language: str, code: str, arguments: dict):
         import requests
 
+        if self.check_if_canceled("CodeExec execution"):
+            return
+
         try:
             code_b64 = self._encode_code(code)
             code_req = CodeExecutionRequest(code_b64=code_b64, language=language, arguments=arguments).model_dump()
         except Exception as e:
+            if self.check_if_canceled("CodeExec execution"):
+                return
+
             self.set_output("_ERROR", "construct code request error: " + str(e))
 
         try:
-            resp = requests.post(url=f"http://{settings.SANDBOX_HOST}:9385/run", json=code_req, timeout=os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))
-            logging.info(f"http://{settings.SANDBOX_HOST}:9385/run", code_req, resp.status_code)
+            if self.check_if_canceled("CodeExec execution"):
+                return "Task has been canceled"
+
+            resp = requests.post(url=f"http://{settings.SANDBOX_HOST}:9385/run", json=code_req, timeout=int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
+            logging.info(f"http://{settings.SANDBOX_HOST}:9385/run,  code_req: {code_req}, resp.status_code {resp.status_code}:")
+
+            if self.check_if_canceled("CodeExec execution"):
+                return "Task has been canceled"
+
             if resp.status_code != 200:
                 resp.raise_for_status()
             body = resp.json()
@@ -173,16 +190,25 @@ class CodeExec(ToolBase, ABC):
                 logging.info(f"http://{settings.SANDBOX_HOST}:9385/run -> {rt}")
                 if isinstance(rt, tuple):
                     for i, (k, o) in enumerate(self._param.outputs.items()):
+                        if self.check_if_canceled("CodeExec execution"):
+                            return
+
                         if k.find("_") == 0:
                             continue
                         o["value"] = rt[i]
                 elif isinstance(rt, dict):
                     for i, (k, o) in enumerate(self._param.outputs.items()):
+                        if self.check_if_canceled("CodeExec execution"):
+                            return
+
                         if k not in rt or k.find("_") == 0:
                             continue
                         o["value"] = rt[k]
                 else:
                     for i, (k, o) in enumerate(self._param.outputs.items()):
+                        if self.check_if_canceled("CodeExec execution"):
+                            return
+
                         if k.find("_") == 0:
                             continue
                         o["value"] = rt
@@ -190,6 +216,9 @@ class CodeExec(ToolBase, ABC):
                 self.set_output("_ERROR", "There is no response from sandbox")
 
         except Exception as e:
+            if self.check_if_canceled("CodeExec execution"):
+                return
+
             self.set_output("_ERROR", "Exception executing code: " + str(e))
 
         return self.output()

agent/tools/crawler.py

@@ -29,7 +29,7 @@ class CrawlerParam(ToolParamBase):
         super().__init__()
         self.proxy = None
         self.extract_type = "markdown"
-    
+
     def check(self):
         self.check_valid_value(self.extract_type, "Type of content from the crawler", ['html', 'markdown', 'content'])
 
@@ -47,18 +47,24 @@ class Crawler(ToolBase, ABC):
             result = asyncio.run(self.get_web(ans))
 
             return Crawler.be_output(result)
-            
+
         except Exception as e:
             return Crawler.be_output(f"An unexpected error occurred: {str(e)}")
 
     async def get_web(self, url):
+        if self.check_if_canceled("Crawler async operation"):
+            return
+
         proxy = self._param.proxy if self._param.proxy else None
         async with AsyncWebCrawler(verbose=True, proxy=proxy) as crawler:
             result = await crawler.arun(
                 url=url,
                 bypass_cache=True
             )
-            
+
+            if self.check_if_canceled("Crawler async operation"):
+                return
+
             if self._param.extract_type == 'html':
                 return result.cleaned_html
             elif self._param.extract_type == 'markdown':

agent/tools/deepl.py

@@ -46,11 +46,16 @@ class DeepL(ComponentBase, ABC):
     component_name = "DeepL"
 
     def _run(self, history, **kwargs):
+        if self.check_if_canceled("DeepL processing"):
+            return
         ans = self.get_input()
         ans = " - ".join(ans["content"]) if "content" in ans else ""
         if not ans:
             return DeepL.be_output("")
 
+        if self.check_if_canceled("DeepL processing"):
+            return
+
         try:
             translator = deepl.Translator(self._param.auth_key)
             result = translator.translate_text(ans, source_lang=self._param.source_lang,
@@ -58,4 +63,6 @@ class DeepL(ComponentBase, ABC):
 
             return DeepL.be_output(result.text)
         except Exception as e:
+            if self.check_if_canceled("DeepL processing"):
+                return
             DeepL.be_output("**Error**:" + str(e))

agent/tools/duckduckgo.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 from duckduckgo_search import DDGS
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class DuckDuckGoParam(ToolParamBase):
@@ -73,19 +73,32 @@ class DuckDuckGoParam(ToolParamBase):
 class DuckDuckGo(ToolBase, ABC):
     component_name = "DuckDuckGo"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("DuckDuckGo processing"):
+            return
+
         if not kwargs.get("query"):
             self.set_output("formalized_content", "")
             return ""
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("DuckDuckGo processing"):
+                return
+
             try:
                 if kwargs.get("topic", "general") == "general":
                     with DDGS() as ddgs:
+                        if self.check_if_canceled("DuckDuckGo processing"):
+                            return
+
                         # {'title': '', 'href': '', 'body': ''}
                         duck_res = ddgs.text(kwargs["query"], max_results=self._param.top_n)
+
+                        if self.check_if_canceled("DuckDuckGo processing"):
+                            return
+
                         self._retrieve_chunks(duck_res,
                                               get_title=lambda r: r["title"],
                                               get_url=lambda r: r.get("href", r.get("url")),
@@ -94,8 +107,15 @@ class DuckDuckGo(ToolBase, ABC):
                         return self.output("formalized_content")
                 else:
                     with DDGS() as ddgs:
+                        if self.check_if_canceled("DuckDuckGo processing"):
+                            return
+
                         # {'date': '', 'title': '', 'body': '', 'url': '', 'image': '', 'source': ''}
                         duck_res = ddgs.news(kwargs["query"], max_results=self._param.top_n)
+
+                        if self.check_if_canceled("DuckDuckGo processing"):
+                            return
+
                         self._retrieve_chunks(duck_res,
                                               get_title=lambda r: r["title"],
                                               get_url=lambda r: r.get("href", r.get("url")),
@@ -103,6 +123,9 @@ class DuckDuckGo(ToolBase, ABC):
                         self.set_output("json", duck_res)
                         return self.output("formalized_content")
             except Exception as e:
+                if self.check_if_canceled("DuckDuckGo processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"DuckDuckGo error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -115,6 +138,6 @@ class DuckDuckGo(ToolBase, ABC):
 
     def thoughts(self) -> str:
         return """
-Keywords: {} 
+Keywords: {}
 Looking for the most relevant articles.
-                """.format(self.get_input().get("query", "-_-!"))
+                """.format(self.get_input().get("query", "-_-!"))

agent/tools/email.py

@@ -25,7 +25,7 @@ from email.header import Header
 from email.utils import formataddr
 
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class EmailParam(ToolParamBase):
@@ -98,22 +98,30 @@ class EmailParam(ToolParamBase):
 
 class Email(ToolBase, ABC):
     component_name = "Email"
-    
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 60))
+
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Email processing"):
+            return
+
         if not kwargs.get("to_email"):
             self.set_output("success", False)
             return ""
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("Email processing"):
+                return
+
             try:
                 # Parse JSON string passed from upstream
                 email_data = kwargs
 
                 # Validate required fields
                 if "to_email" not in email_data:
-                    return Email.be_output("Missing required field: to_email")
+                    self.set_output("_ERROR", "Missing required field: to_email")
+                    self.set_output("success", False)
+                    return False
 
                 # Create email object
                 msg = MIMEMultipart('alternative')
@@ -133,6 +141,9 @@ class Email(ToolBase, ABC):
                 # Connect to SMTP server and send
                 logging.info(f"Connecting to SMTP server {self._param.smtp_server}:{self._param.smtp_port}")
 
+                if self.check_if_canceled("Email processing"):
+                    return
+
                 context = smtplib.ssl.create_default_context()
                 with smtplib.SMTP(self._param.smtp_server, self._param.smtp_port) as server:
                     server.ehlo()
@@ -149,6 +160,10 @@ class Email(ToolBase, ABC):
 
                     # Send email
                     logging.info(f"Sending email to recipients: {recipients}")
+
+                    if self.check_if_canceled("Email processing"):
+                        return
+
                     try:
                         server.send_message(msg, self._param.email, recipients)
                         success = True
@@ -212,4 +227,4 @@ class Email(ToolBase, ABC):
 To: {}
 Subject: {}
 Your email is on its way—sit tight!
-""".format(inputs.get("to_email", "-_-!"), inputs.get("subject", "-_-!"))
+""".format(inputs.get("to_email", "-_-!"), inputs.get("subject", "-_-!"))

agent/tools/exesql.py

@@ -22,7 +22,7 @@ import pymysql
 import psycopg2
 import pyodbc
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class ExeSQLParam(ToolParamBase):
@@ -53,12 +53,13 @@ class ExeSQLParam(ToolParamBase):
         self.max_records = 1024
 
     def check(self):
-        self.check_valid_value(self.db_type, "Choose DB type", ['mysql', 'postgresql', 'mariadb', 'mssql'])
+        self.check_valid_value(self.db_type, "Choose DB type", ['mysql', 'postgres', 'mariadb', 'mssql', 'IBM DB2', 'trino'])
         self.check_empty(self.database, "Database name")
         self.check_empty(self.username, "database username")
         self.check_empty(self.host, "IP Address")
         self.check_positive_integer(self.port, "IP Port")
-        self.check_empty(self.password, "Database password")
+        if self.db_type != "trino":
+            self.check_empty(self.password, "Database password")
         self.check_positive_integer(self.max_records, "Maximum number of records")
         if self.database == "rag_flow":
             if self.host == "ragflow-mysql":
@@ -78,8 +79,10 @@ class ExeSQLParam(ToolParamBase):
 class ExeSQL(ToolBase, ABC):
     component_name = "ExeSQL"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("ExeSQL processing"):
+            return
 
         def convert_decimals(obj):
             from decimal import Decimal
@@ -95,6 +98,9 @@ class ExeSQL(ToolBase, ABC):
         if not sql:
             raise Exception("SQL for `ExeSQL` MUST not be empty.")
 
+        if self.check_if_canceled("ExeSQL processing"):
+            return
+
         vars = self.get_input_elements_from_text(sql)
         args = {}
         for k, o in vars.items():
@@ -107,11 +113,14 @@ class ExeSQL(ToolBase, ABC):
             self.set_input_value(k, args[k])
         sql = self.string_format(sql, args)
 
+        if self.check_if_canceled("ExeSQL processing"):
+            return
+
         sqls = sql.split(";")
         if self._param.db_type in ["mysql", "mariadb"]:
             db = pymysql.connect(db=self._param.database, user=self._param.username, host=self._param.host,
                                  port=self._param.port, password=self._param.password)
-        elif self._param.db_type == 'postgresql':
+        elif self._param.db_type == 'postgres':
             db = psycopg2.connect(dbname=self._param.database, user=self._param.username, host=self._param.host,
                                   port=self._param.port, password=self._param.password)
         elif self._param.db_type == 'mssql':
@@ -123,6 +132,101 @@ class ExeSQL(ToolBase, ABC):
                     r'PWD=' + self._param.password
             )
             db = pyodbc.connect(conn_str)
+        elif self._param.db_type == 'trino':
+            try:
+                import trino
+                from trino.auth import BasicAuthentication
+            except Exception:
+                raise Exception("Missing dependency 'trino'. Please install: pip install trino")
+
+            def _parse_catalog_schema(db: str):
+                if not db:
+                    return None, None
+                if "." in db:
+                    c, s = db.split(".", 1)
+                elif "/" in db:
+                    c, s = db.split("/", 1)
+                else:
+                    c, s = db, "default"
+                return c, s
+
+            catalog, schema = _parse_catalog_schema(self._param.database)
+            if not catalog:
+                raise Exception("For Trino, `database` must be 'catalog.schema' or at least 'catalog'.")
+
+            http_scheme = "https" if os.environ.get("TRINO_USE_TLS", "0") == "1" else "http"
+            auth = None
+            if http_scheme == "https" and self._param.password:
+                auth = BasicAuthentication(self._param.username, self._param.password)
+
+            try:
+                db = trino.dbapi.connect(
+                    host=self._param.host,
+                    port=int(self._param.port or 8080),
+                    user=self._param.username or "ragflow",
+                    catalog=catalog,
+                    schema=schema or "default",
+                    http_scheme=http_scheme,
+                    auth=auth
+                )
+            except Exception as e:
+                raise Exception("Database Connection Failed! \n" + str(e))
+        elif self._param.db_type == 'IBM DB2':
+            import ibm_db
+            conn_str = (
+                f"DATABASE={self._param.database};"
+                f"HOSTNAME={self._param.host};"
+                f"PORT={self._param.port};"
+                f"PROTOCOL=TCPIP;"
+                f"UID={self._param.username};"
+                f"PWD={self._param.password};"
+            )
+            try:
+                conn = ibm_db.connect(conn_str, "", "")
+            except Exception as e:
+                raise Exception("Database Connection Failed! \n" + str(e))
+
+            sql_res = []
+            formalized_content = []
+            for single_sql in sqls:
+                if self.check_if_canceled("ExeSQL processing"):
+                    ibm_db.close(conn)
+                    return
+
+                single_sql = single_sql.replace("```", "").strip()
+                if not single_sql:
+                    continue
+                single_sql = re.sub(r"\[ID:[0-9]+\]", "", single_sql)
+
+                stmt = ibm_db.exec_immediate(conn, single_sql)
+                rows = []
+                row = ibm_db.fetch_assoc(stmt)
+                while row and len(rows) < self._param.max_records:
+                    if self.check_if_canceled("ExeSQL processing"):
+                        ibm_db.close(conn)
+                        return
+                    rows.append(row)
+                    row = ibm_db.fetch_assoc(stmt)
+
+                if not rows:
+                    sql_res.append({"content": "No record in the database!"})
+                    continue
+
+                df = pd.DataFrame(rows)
+                for col in df.columns:
+                    if pd.api.types.is_datetime64_any_dtype(df[col]):
+                        df[col] = df[col].dt.strftime("%Y-%m-%d")
+
+                df = df.where(pd.notnull(df), None)
+
+                sql_res.append(convert_decimals(df.to_dict(orient="records")))
+                formalized_content.append(df.to_markdown(index=False, floatfmt=".6f"))
+
+            ibm_db.close(conn)
+
+            self.set_output("json", sql_res)
+            self.set_output("formalized_content", "\n\n".join(formalized_content))
+            return self.output("formalized_content")
         try:
             cursor = db.cursor()
         except Exception as e:
@@ -131,6 +235,11 @@ class ExeSQL(ToolBase, ABC):
         sql_res = []
         formalized_content = []
         for single_sql in sqls:
+            if self.check_if_canceled("ExeSQL processing"):
+                cursor.close()
+                db.close()
+                return
+
             single_sql = single_sql.replace('```','')
             if not single_sql:
                 continue
@@ -150,9 +259,14 @@ class ExeSQL(ToolBase, ABC):
                 if pd.api.types.is_datetime64_any_dtype(single_res[col]):
                     single_res[col] = single_res[col].dt.strftime('%Y-%m-%d')
 
+            single_res = single_res.where(pd.notnull(single_res), None)
+
             sql_res.append(convert_decimals(single_res.to_dict(orient='records')))
             formalized_content.append(single_res.to_markdown(index=False, floatfmt=".6f"))
 
+        cursor.close()
+        db.close()
+
         self.set_output("json", sql_res)
         self.set_output("formalized_content", "\n\n".join(formalized_content))
         return self.output("formalized_content")

agent/tools/github.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 import requests
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class GitHubParam(ToolParamBase):
@@ -57,19 +57,29 @@ class GitHubParam(ToolParamBase):
 class GitHub(ToolBase, ABC):
     component_name = "GitHub"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("GitHub processing"):
+            return
+
         if not kwargs.get("query"):
             self.set_output("formalized_content", "")
             return ""
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("GitHub processing"):
+                return
+
             try:
                 url = 'https://api.github.com/search/repositories?q=' + kwargs["query"] + '&sort=stars&order=desc&per_page=' + str(
                     self._param.top_n)
                 headers = {"Content-Type": "application/vnd.github+json", "X-GitHub-Api-Version": '2022-11-28'}
                 response = requests.get(url=url, headers=headers).json()
+
+                if self.check_if_canceled("GitHub processing"):
+                    return
+
                 self._retrieve_chunks(response['items'],
                                       get_title=lambda r: r["name"],
                                       get_url=lambda r: r["html_url"],
@@ -77,6 +87,9 @@ class GitHub(ToolBase, ABC):
                 self.set_output("json", response['items'])
                 return self.output("formalized_content")
             except Exception as e:
+                if self.check_if_canceled("GitHub processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"GitHub error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -88,4 +101,4 @@ class GitHub(ToolBase, ABC):
         assert False, self.output()
 
     def thoughts(self) -> str:
-        return "Scanning GitHub repos related to `{}`.".format(self.get_input().get("query", "-_-!"))
+        return "Scanning GitHub repos related to `{}`.".format(self.get_input().get("query", "-_-!"))

agent/tools/google.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 from serpapi import GoogleSearch
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class GoogleParam(ToolParamBase):
@@ -116,8 +116,11 @@ class GoogleParam(ToolParamBase):
 class Google(ToolBase, ABC):
     component_name = "Google"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Google processing"):
+            return
+
         if not kwargs.get("q"):
             self.set_output("formalized_content", "")
             return ""
@@ -132,8 +135,15 @@ class Google(ToolBase, ABC):
         }
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("Google processing"):
+                return
+
             try:
                 search = GoogleSearch(params).get_dict()
+
+                if self.check_if_canceled("Google processing"):
+                    return
+
                 self._retrieve_chunks(search["organic_results"],
                                       get_title=lambda r: r["title"],
                                       get_url=lambda r: r["link"],
@@ -142,6 +152,9 @@ class Google(ToolBase, ABC):
                 self.set_output("json", search["organic_results"])
                 return self.output("formalized_content")
             except Exception as e:
+                if self.check_if_canceled("Google processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"Google error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -154,6 +167,6 @@ class Google(ToolBase, ABC):
 
     def thoughts(self) -> str:
         return """
-Keywords: {} 
+Keywords: {}
 Looking for the most relevant articles.
-        """.format(self.get_input().get("query", "-_-!"))
+        """.format(self.get_input().get("query", "-_-!"))

agent/tools/googlescholar.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 from scholarly import scholarly
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class GoogleScholarParam(ToolParamBase):
@@ -63,17 +63,27 @@ class GoogleScholarParam(ToolParamBase):
 class GoogleScholar(ToolBase, ABC):
     component_name = "GoogleScholar"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("GoogleScholar processing"):
+            return
+
         if not kwargs.get("query"):
             self.set_output("formalized_content", "")
             return ""
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("GoogleScholar processing"):
+                return
+
             try:
                 scholar_client = scholarly.search_pubs(kwargs["query"], patents=self._param.patents, year_low=self._param.year_low,
                                                        year_high=self._param.year_high, sort_by=self._param.sort_by)
+
+                if self.check_if_canceled("GoogleScholar processing"):
+                    return
+
                 self._retrieve_chunks(scholar_client,
                                       get_title=lambda r: r['bib']['title'],
                                       get_url=lambda r: r["pub_url"],
@@ -82,6 +92,9 @@ class GoogleScholar(ToolBase, ABC):
                 self.set_output("json", list(scholar_client))
                 return self.output("formalized_content")
             except Exception as e:
+                if self.check_if_canceled("GoogleScholar processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"GoogleScholar error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -93,4 +106,4 @@ class GoogleScholar(ToolBase, ABC):
         assert False, self.output()
 
     def thoughts(self) -> str:
-        return "Looking for scholarly papers on `{}`,” prioritising reputable sources.".format(self.get_input().get("query", "-_-!"))
+        return "Looking for scholarly papers on `{}`,” prioritising reputable sources.".format(self.get_input().get("query", "-_-!"))

agent/tools/jin10.py

@@ -50,6 +50,9 @@ class Jin10(ComponentBase, ABC):
     component_name = "Jin10"
 
     def _run(self, history, **kwargs):
+        if self.check_if_canceled("Jin10 processing"):
+            return
+
         ans = self.get_input()
         ans = " - ".join(ans["content"]) if "content" in ans else ""
         if not ans:
@@ -58,6 +61,9 @@ class Jin10(ComponentBase, ABC):
         jin10_res = []
         headers = {'secret-key': self._param.secret_key}
         try:
+            if self.check_if_canceled("Jin10 processing"):
+                return
+
             if self._param.type == "flash":
                 params = {
                     'category': self._param.flash_type,
@@ -69,6 +75,8 @@ class Jin10(ComponentBase, ABC):
                     headers=headers, data=json.dumps(params))
                 response = response.json()
                 for i in response['data']:
+                    if self.check_if_canceled("Jin10 processing"):
+                        return
                     jin10_res.append({"content": i['data']['content']})
             if self._param.type == "calendar":
                 params = {
@@ -79,6 +87,8 @@ class Jin10(ComponentBase, ABC):
                     headers=headers, data=json.dumps(params))
 
                 response = response.json()
+                if self.check_if_canceled("Jin10 processing"):
+                    return
                 jin10_res.append({"content": pd.DataFrame(response['data']).to_markdown()})
             if self._param.type == "symbols":
                 params = {
@@ -90,8 +100,12 @@ class Jin10(ComponentBase, ABC):
                     url='https://open-data-api.jin10.com/data-api/' + self._param.symbols_datatype + '?type=' + self._param.symbols_type,
                     headers=headers, data=json.dumps(params))
                 response = response.json()
+                if self.check_if_canceled("Jin10 processing"):
+                    return
                 if self._param.symbols_datatype == "symbols":
                     for i in response['data']:
+                        if self.check_if_canceled("Jin10 processing"):
+                            return
                         i['Commodity Code'] = i['c']
                         i['Stock Exchange'] = i['e']
                         i['Commodity Name'] = i['n']
@@ -99,6 +113,8 @@ class Jin10(ComponentBase, ABC):
                         del i['c'], i['e'], i['n'], i['t']
                 if self._param.symbols_datatype == "quotes":
                     for i in response['data']:
+                        if self.check_if_canceled("Jin10 processing"):
+                            return
                         i['Selling Price'] = i['a']
                         i['Buying Price'] = i['b']
                         i['Commodity Code'] = i['c']
@@ -120,8 +136,12 @@ class Jin10(ComponentBase, ABC):
                     url='https://open-data-api.jin10.com/data-api/news',
                     headers=headers, data=json.dumps(params))
                 response = response.json()
+                if self.check_if_canceled("Jin10 processing"):
+                    return
                 jin10_res.append({"content": pd.DataFrame(response['data']).to_markdown()})
         except Exception as e:
+            if self.check_if_canceled("Jin10 processing"):
+                return
             return Jin10.be_output("**ERROR**: " + str(e))
 
         if not jin10_res:

agent/tools/pubmed.py

@@ -21,7 +21,7 @@ from Bio import Entrez
 import re
 import xml.etree.ElementTree as ET
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class PubMedParam(ToolParamBase):
@@ -33,7 +33,7 @@ class PubMedParam(ToolParamBase):
         self.meta:ToolMeta = {
             "name": "pubmed_search",
             "description": """
-PubMed is an openly accessible, free database which includes primarily the MEDLINE database of references and abstracts on life sciences and biomedical topics. 
+PubMed is an openly accessible, free database which includes primarily the MEDLINE database of references and abstracts on life sciences and biomedical topics.
 In addition to MEDLINE, PubMed provides access to:
  - older references from the print version of Index Medicus, back to 1951 and earlier
  - references to some journals before they were indexed in Index Medicus and MEDLINE, for instance Science, BMJ, and Annals of Surgery
@@ -69,31 +69,42 @@ In addition to MEDLINE, PubMed provides access to:
 class PubMed(ToolBase, ABC):
     component_name = "PubMed"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("PubMed processing"):
+            return
+
         if not kwargs.get("query"):
             self.set_output("formalized_content", "")
             return ""
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("PubMed processing"):
+                return
+
             try:
                 Entrez.email = self._param.email
                 pubmedids = Entrez.read(Entrez.esearch(db='pubmed', retmax=self._param.top_n, term=kwargs["query"]))['IdList']
+
+                if self.check_if_canceled("PubMed processing"):
+                    return
+
                 pubmedcnt = ET.fromstring(re.sub(r'<(/?)b>|<(/?)i>', '', Entrez.efetch(db='pubmed', id=",".join(pubmedids),
                                                                                        retmode="xml").read().decode("utf-8")))
+
+                if self.check_if_canceled("PubMed processing"):
+                    return
+
                 self._retrieve_chunks(pubmedcnt.findall("PubmedArticle"),
                                       get_title=lambda child: child.find("MedlineCitation").find("Article").find("ArticleTitle").text,
                                       get_url=lambda child: "https://pubmed.ncbi.nlm.nih.gov/" + child.find("MedlineCitation").find("PMID").text,
-                                      get_content=lambda child: child.find("MedlineCitation") \
-                                                                    .find("Article") \
-                                                                    .find("Abstract") \
-                                                                    .find("AbstractText").text \
-                                                                    if child.find("MedlineCitation")\
-                                                                            .find("Article").find("Abstract")  \
-                                                                    else "No abstract available")
+                                      get_content=lambda child: self._format_pubmed_content(child),)
                 return self.output("formalized_content")
             except Exception as e:
+                if self.check_if_canceled("PubMed processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"PubMed error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -104,5 +115,50 @@ class PubMed(ToolBase, ABC):
 
         assert False, self.output()
 
+    def _format_pubmed_content(self, child):
+        """Extract structured reference info from PubMed XML"""
+        def safe_find(path):
+            node = child
+            for p in path.split("/"):
+                if node is None:
+                    return None
+                node = node.find(p)
+            return node.text if node is not None and node.text else None
+
+        title = safe_find("MedlineCitation/Article/ArticleTitle") or "No title"
+        abstract = safe_find("MedlineCitation/Article/Abstract/AbstractText") or "No abstract available"
+        journal = safe_find("MedlineCitation/Article/Journal/Title") or "Unknown Journal"
+        volume = safe_find("MedlineCitation/Article/Journal/JournalIssue/Volume") or "-"
+        issue = safe_find("MedlineCitation/Article/Journal/JournalIssue/Issue") or "-"
+        pages = safe_find("MedlineCitation/Article/Pagination/MedlinePgn") or "-"
+
+        # Authors
+        authors = []
+        for author in child.findall(".//AuthorList/Author"):
+            lastname = safe_find("LastName") or ""
+            forename = safe_find("ForeName") or ""
+            fullname = f"{forename} {lastname}".strip()
+            if fullname:
+                authors.append(fullname)
+        authors_str = ", ".join(authors) if authors else "Unknown Authors"
+
+        # DOI
+        doi = None
+        for eid in child.findall(".//ArticleId"):
+            if eid.attrib.get("IdType") == "doi":
+                doi = eid.text
+                break
+
+        return (
+            f"Title: {title}\n"
+            f"Authors: {authors_str}\n"
+            f"Journal: {journal}\n"
+            f"Volume: {volume}\n"
+            f"Issue: {issue}\n"
+            f"Pages: {pages}\n"
+            f"DOI: {doi or '-'}\n"
+            f"Abstract: {abstract.strip()}"
+        )
+
     def thoughts(self) -> str:
-        return "Looking for scholarly papers on `{}`,” prioritising reputable sources.".format(self.get_input().get("query", "-_-!"))
+        return "Looking for scholarly papers on `{}`,” prioritising reputable sources.".format(self.get_input().get("query", "-_-!"))

agent/tools/qweather.py

@@ -58,12 +58,18 @@ class QWeather(ComponentBase, ABC):
     component_name = "QWeather"
 
     def _run(self, history, **kwargs):
+        if self.check_if_canceled("Qweather processing"):
+            return
+
         ans = self.get_input()
         ans = "".join(ans["content"]) if "content" in ans else ""
         if not ans:
             return QWeather.be_output("")
 
         try:
+            if self.check_if_canceled("Qweather processing"):
+                return
+
             response = requests.get(
                 url="https://geoapi.qweather.com/v2/city/lookup?location=" + ans + "&key=" + self._param.web_apikey).json()
             if response["code"] == "200":
@@ -71,16 +77,23 @@ class QWeather(ComponentBase, ABC):
             else:
                 return QWeather.be_output("**Error**" + self._param.error_code[response["code"]])
 
+            if self.check_if_canceled("Qweather processing"):
+                return
+
             base_url = "https://api.qweather.com/v7/" if self._param.user_type == 'paid' else "https://devapi.qweather.com/v7/"
 
             if self._param.type == "weather":
                 url = base_url + "weather/" + self._param.time_period + "?location=" + location_id + "&key=" + self._param.web_apikey + "&lang=" + self._param.lang
                 response = requests.get(url=url).json()
+                if self.check_if_canceled("Qweather processing"):
+                    return
                 if response["code"] == "200":
                     if self._param.time_period == "now":
                         return QWeather.be_output(str(response["now"]))
                     else:
                         qweather_res = [{"content": str(i) + "\n"} for i in response["daily"]]
+                        if self.check_if_canceled("Qweather processing"):
+                            return
                         if not qweather_res:
                             return QWeather.be_output("")
 
@@ -92,6 +105,8 @@ class QWeather(ComponentBase, ABC):
             elif self._param.type == "indices":
                 url = base_url + "indices/1d?type=0&location=" + location_id + "&key=" + self._param.web_apikey + "&lang=" + self._param.lang
                 response = requests.get(url=url).json()
+                if self.check_if_canceled("Qweather processing"):
+                    return
                 if response["code"] == "200":
                     indices_res = response["daily"][0]["date"] + "\n" + "\n".join(
                         [i["name"] + ": " + i["category"] + ", " + i["text"] for i in response["daily"]])
@@ -103,9 +118,13 @@ class QWeather(ComponentBase, ABC):
             elif self._param.type == "airquality":
                 url = base_url + "air/now?location=" + location_id + "&key=" + self._param.web_apikey + "&lang=" + self._param.lang
                 response = requests.get(url=url).json()
+                if self.check_if_canceled("Qweather processing"):
+                    return
                 if response["code"] == "200":
                     return QWeather.be_output(str(response["now"]))
                 else:
                     return QWeather.be_output("**Error**" + self._param.error_code[response["code"]])
         except Exception as e:
+            if self.check_if_canceled("Qweather processing"):
+                return
             return QWeather.be_output("**Error**" + str(e))

agent/tools/retrieval.py

@@ -13,18 +13,21 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+from functools import partial
+import json
 import os
 import re
 from abc import ABC
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api.db import LLMType
+from common.constants import LLMType
+from api.db.services.document_service import DocumentService
+from api.db.services.dialog_service import meta_filter
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
-from api import settings
-from api.utils.api_utils import timeout
+from common import settings
+from common.connection_utils import timeout
 from rag.app.tag import label_question
-from rag.prompts import kb_prompt
-from rag.prompts.prompts import cross_languages
+from rag.prompts.generator import cross_languages, kb_prompt, gen_meta_filter
 
 
 class RetrievalParam(ToolParamBase):
@@ -58,6 +61,8 @@ class RetrievalParam(ToolParamBase):
         self.empty_response = ""
         self.use_kg = False
         self.cross_languages = []
+        self.toc_enhance = False
+        self.meta_data_filter={}
 
     def check(self):
         self.check_decimal_float(self.similarity_threshold, "[Retrieval] Similarity threshold")
@@ -75,10 +80,14 @@ class RetrievalParam(ToolParamBase):
 class Retrieval(ToolBase, ABC):
     component_name = "Retrieval"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Retrieval processing"):
+            return
+
         if not kwargs.get("query"):
             self.set_output("formalized_content", self._param.empty_response)
+            return
 
         kb_ids: list[str] = []
         for id in self._param.kb_ids:
@@ -117,12 +126,55 @@ class Retrieval(ToolBase, ABC):
         vars = self.get_input_elements_from_text(kwargs["query"])
         vars = {k:o["value"] for k,o in vars.items()}
         query = self.string_format(kwargs["query"], vars)
+
+        doc_ids=[]
+        if self._param.meta_data_filter!={}:
+            metas = DocumentService.get_meta_by_kbs(kb_ids)
+            if self._param.meta_data_filter.get("method") == "auto":
+                chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT)
+                filters = gen_meta_filter(chat_mdl, metas, query)
+                doc_ids.extend(meta_filter(metas, filters))
+                if not doc_ids:
+                    doc_ids = None
+            elif self._param.meta_data_filter.get("method") == "manual":
+                filters=self._param.meta_data_filter["manual"]
+                for flt in filters:
+                    pat = re.compile(self.variable_ref_patt)
+                    s = flt["value"]
+                    out_parts = []
+                    last = 0
+
+                    for m in pat.finditer(s):
+                        out_parts.append(s[last:m.start()])
+                        key = m.group(1)
+                        v = self._canvas.get_variable_value(key)
+                        if v is None:
+                            rep = ""
+                        elif isinstance(v, partial):
+                            buf = []
+                            for chunk in v():
+                                buf.append(chunk)
+                            rep = "".join(buf)
+                        elif isinstance(v, str):
+                            rep = v
+                        else:
+                            rep = json.dumps(v, ensure_ascii=False)
+
+                        out_parts.append(rep)
+                        last = m.end()
+
+                    out_parts.append(s[last:])
+                    flt["value"] = "".join(out_parts)
+                doc_ids.extend(meta_filter(metas, filters))
+                if not doc_ids:
+                    doc_ids = None
+
         if self._param.cross_languages:
             query = cross_languages(kbs[0].tenant_id, None, query, self._param.cross_languages)
 
         if kbs:
             query = re.sub(r"^user[:：\s]*", "", query, flags=re.IGNORECASE)
-            kbinfos = settings.retrievaler.retrieval(
+            kbinfos = settings.retriever.retrieval(
                 query,
                 embd_mdl,
                 [kb.tenant_id for kb in kbs],
@@ -131,23 +183,38 @@ class Retrieval(ToolBase, ABC):
                 self._param.top_n,
                 self._param.similarity_threshold,
                 1 - self._param.keywords_similarity_weight,
+                doc_ids=doc_ids,
                 aggs=False,
                 rerank_mdl=rerank_mdl,
                 rank_feature=label_question(query, kbs),
             )
+            if self.check_if_canceled("Retrieval processing"):
+                return
+
+            if self._param.toc_enhance:
+                chat_mdl = LLMBundle(self._canvas._tenant_id, LLMType.CHAT)
+                cks = settings.retriever.retrieval_by_toc(query, kbinfos["chunks"], [kb.tenant_id for kb in kbs], chat_mdl, self._param.top_n)
+                if self.check_if_canceled("Retrieval processing"):
+                    return
+                if cks:
+                    kbinfos["chunks"] = cks
             if self._param.use_kg:
-                ck = settings.kg_retrievaler.retrieval(query,
+                ck = settings.kg_retriever.retrieval(query,
                                                        [kb.tenant_id for kb in kbs],
                                                        kb_ids,
                                                        embd_mdl,
                                                        LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT))
+                if self.check_if_canceled("Retrieval processing"):
+                    return
                 if ck["content_with_weight"]:
                     kbinfos["chunks"].insert(0, ck)
         else:
             kbinfos = {"chunks": [], "doc_aggs": []}
 
         if self._param.use_kg and kbs:
-            ck = settings.kg_retrievaler.retrieval(query, [kb.tenant_id for kb in kbs], filtered_kb_ids, embd_mdl, LLMBundle(kbs[0].tenant_id, LLMType.CHAT))
+            ck = settings.kg_retriever.retrieval(query, [kb.tenant_id for kb in kbs], filtered_kb_ids, embd_mdl, LLMBundle(kbs[0].tenant_id, LLMType.CHAT))
+            if self.check_if_canceled("Retrieval processing"):
+                return
             if ck["content_with_weight"]:
                 ck["content"] = ck["content_with_weight"]
                 del ck["content_with_weight"]
@@ -163,13 +230,20 @@ class Retrieval(ToolBase, ABC):
             self.set_output("formalized_content", self._param.empty_response)
             return
 
+        # Format the chunks for JSON output (similar to how other tools do it)
+        json_output = kbinfos["chunks"].copy()
+
         self._canvas.add_reference(kbinfos["chunks"], kbinfos["doc_aggs"])
         form_cnt = "\n".join(kb_prompt(kbinfos, 200000, True))
+
+        # Set both formalized content and JSON output
         self.set_output("formalized_content", form_cnt)
+        self.set_output("json", json_output)
+
         return form_cnt
 
     def thoughts(self) -> str:
         return """
-Keywords: {} 
+Keywords: {}
 Looking for the most relevant articles.
-        """.format(self.get_input().get("query", "-_-!"))
+        """.format(self.get_input().get("query", "-_-!"))

agent/tools/searxng.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 import requests
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class SearXNGParam(ToolParamBase):
@@ -77,15 +77,18 @@ class SearXNGParam(ToolParamBase):
 class SearXNG(ToolBase, ABC):
     component_name = "SearXNG"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("SearXNG processing"):
+            return
+
         # Gracefully handle try-run without inputs
         query = kwargs.get("query")
         if not query or not isinstance(query, str) or not query.strip():
             self.set_output("formalized_content", "")
             return ""
 
-        searxng_url = (kwargs.get("searxng_url") or getattr(self._param, "searxng_url", "") or "").strip()
+        searxng_url = (getattr(self._param, "searxng_url", "") or kwargs.get("searxng_url") or "").strip()
         # In try-run, if no URL configured, just return empty instead of raising
         if not searxng_url:
             self.set_output("formalized_content", "")
@@ -93,8 +96,10 @@ class SearXNG(ToolBase, ABC):
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("SearXNG processing"):
+                return
+
             try:
-                # 构建搜索参数
                 search_params = {
                     'q': query,
                     'format': 'json',
@@ -104,41 +109,49 @@ class SearXNG(ToolBase, ABC):
                     'pageno': 1
                 }
 
-                # 发送搜索请求
                 response = requests.get(
                     f"{searxng_url}/search",
                     params=search_params,
                     timeout=10
                 )
                 response.raise_for_status()
-                
+
+                if self.check_if_canceled("SearXNG processing"):
+                    return
+
                 data = response.json()
-                
-                # 验证响应数据
+
                 if not data or not isinstance(data, dict):
                     raise ValueError("Invalid response from SearXNG")
-                
+
                 results = data.get("results", [])
                 if not isinstance(results, list):
                     raise ValueError("Invalid results format from SearXNG")
-                
-                # 限制结果数量
+
                 results = results[:self._param.top_n]
-                
-                # 处理搜索结果
+
+                if self.check_if_canceled("SearXNG processing"):
+                    return
+
                 self._retrieve_chunks(results,
                                       get_title=lambda r: r.get("title", ""),
                                       get_url=lambda r: r.get("url", ""),
                                       get_content=lambda r: r.get("content", ""))
-                
+
                 self.set_output("json", results)
                 return self.output("formalized_content")
 
             except requests.RequestException as e:
+                if self.check_if_canceled("SearXNG processing"):
+                    return
+
                 last_e = f"Network error: {e}"
                 logging.exception(f"SearXNG network error: {e}")
                 time.sleep(self._param.delay_after_error)
             except Exception as e:
+                if self.check_if_canceled("SearXNG processing"):
+                    return
+
                 last_e = str(e)
                 logging.exception(f"SearXNG error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -151,6 +164,6 @@ class SearXNG(ToolBase, ABC):
 
     def thoughts(self) -> str:
         return """
-Keywords: {} 
+Keywords: {}
 Searching with SearXNG for relevant results...
                 """.format(self.get_input().get("query", "-_-!"))

agent/tools/tavily.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 from tavily import TavilyClient
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class TavilySearchParam(ToolParamBase):
@@ -31,7 +31,7 @@ class TavilySearchParam(ToolParamBase):
         self.meta:ToolMeta = {
             "name": "tavily_search",
             "description": """
-Tavily is a search engine optimized for LLMs, aimed at efficient, quick and persistent search results. 
+Tavily is a search engine optimized for LLMs, aimed at efficient, quick and persistent search results.
 When searching:
    - Start with specific query which should focus on just a single aspect.
    - Number of keywords in query should be less than 5.
@@ -101,8 +101,11 @@ When searching:
 class TavilySearch(ToolBase, ABC):
     component_name = "TavilySearch"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("TavilySearch processing"):
+            return
+
         if not kwargs.get("query"):
             self.set_output("formalized_content", "")
             return ""
@@ -113,10 +116,16 @@ class TavilySearch(ToolBase, ABC):
             if fld not in kwargs:
                 kwargs[fld] = getattr(self._param, fld)
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("TavilySearch processing"):
+                return
+
             try:
                 kwargs["include_images"] = False
                 kwargs["include_raw_content"] = False
                 res = self.tavily_client.search(**kwargs)
+                if self.check_if_canceled("TavilySearch processing"):
+                    return
+
                 self._retrieve_chunks(res["results"],
                                       get_title=lambda r: r["title"],
                                       get_url=lambda r: r["url"],
@@ -125,6 +134,9 @@ class TavilySearch(ToolBase, ABC):
                 self.set_output("json", res["results"])
                 return self.output("formalized_content")
             except Exception as e:
+                if self.check_if_canceled("TavilySearch processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"Tavily error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -136,7 +148,7 @@ class TavilySearch(ToolBase, ABC):
 
     def thoughts(self) -> str:
         return """
-Keywords: {} 
+Keywords: {}
 Looking for the most relevant articles.
                 """.format(self.get_input().get("query", "-_-!"))
 
@@ -199,8 +211,11 @@ class TavilyExtractParam(ToolParamBase):
 class TavilyExtract(ToolBase, ABC):
     component_name = "TavilyExtract"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("TavilyExtract processing"):
+            return
+
         self.tavily_client = TavilyClient(api_key=self._param.api_key)
         last_e = None
         for fld in ["urls", "extract_depth", "format"]:
@@ -209,12 +224,21 @@ class TavilyExtract(ToolBase, ABC):
         if kwargs.get("urls") and isinstance(kwargs["urls"], str):
             kwargs["urls"] = kwargs["urls"].split(",")
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("TavilyExtract processing"):
+                return
+
             try:
                 kwargs["include_images"] = False
                 res = self.tavily_client.extract(**kwargs)
+                if self.check_if_canceled("TavilyExtract processing"):
+                    return
+
                 self.set_output("json", res["results"])
                 return self.output("json")
             except Exception as e:
+                if self.check_if_canceled("TavilyExtract processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"Tavily error: {e}")
         if last_e:
@@ -224,4 +248,4 @@ class TavilyExtract(ToolBase, ABC):
         assert False, self.output()
 
     def thoughts(self) -> str:
-        return "Opened {}—pulling out the main text…".format(self.get_input().get("urls", "-_-!"))
+        return "Opened {}—pulling out the main text…".format(self.get_input().get("urls", "-_-!"))

agent/tools/tushare.py

@@ -43,12 +43,18 @@ class TuShare(ComponentBase, ABC):
     component_name = "TuShare"
 
     def _run(self, history, **kwargs):
+        if self.check_if_canceled("TuShare processing"):
+            return
+
         ans = self.get_input()
         ans = ",".join(ans["content"]) if "content" in ans else ""
         if not ans:
             return TuShare.be_output("")
 
         try:
+            if self.check_if_canceled("TuShare processing"):
+                return
+
             tus_res = []
             params = {
                 "api_name": "news",
@@ -58,12 +64,18 @@ class TuShare(ComponentBase, ABC):
             }
             response = requests.post(url="http://api.tushare.pro", data=json.dumps(params).encode('utf-8'))
             response = response.json()
+            if self.check_if_canceled("TuShare processing"):
+                return
             if response['code'] != 0:
                 return TuShare.be_output(response['msg'])
             df = pd.DataFrame(response['data']['items'])
             df.columns = response['data']['fields']
+            if self.check_if_canceled("TuShare processing"):
+                return
             tus_res.append({"content": (df[df['content'].str.contains(self._param.keyword, case=False)]).to_markdown()})
         except Exception as e:
+            if self.check_if_canceled("TuShare processing"):
+                return
             return TuShare.be_output("**ERROR**: " + str(e))
 
         if not tus_res:

agent/tools/wencai.py

@@ -21,7 +21,7 @@ import pandas as pd
 import pywencai
 
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class WenCaiParam(ToolParamBase):
@@ -68,21 +68,33 @@ fund selection platform: through AI technology, is committed to providing excell
 class WenCai(ToolBase, ABC):
     component_name = "WenCai"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("WenCai processing"):
+            return
+
         if not kwargs.get("query"):
             self.set_output("report", "")
             return ""
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("WenCai processing"):
+                return
+
             try:
                 wencai_res = []
                 res = pywencai.get(query=kwargs["query"], query_type=self._param.query_type, perpage=self._param.top_n)
+                if self.check_if_canceled("WenCai processing"):
+                    return
+
                 if isinstance(res, pd.DataFrame):
                     wencai_res.append(res.to_markdown())
                 elif isinstance(res, dict):
                     for item in res.items():
+                        if self.check_if_canceled("WenCai processing"):
+                            return
+
                         if isinstance(item[1], list):
                             wencai_res.append(item[0] + "\n" + pd.DataFrame(item[1]).to_markdown())
                         elif isinstance(item[1], str):
@@ -100,6 +112,9 @@ class WenCai(ToolBase, ABC):
                 self.set_output("report", "\n\n".join(wencai_res))
                 return self.output("report")
             except Exception as e:
+                if self.check_if_canceled("WenCai processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"WenCai error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -111,4 +126,4 @@ class WenCai(ToolBase, ABC):
         assert False, self.output()
 
     def thoughts(self) -> str:
-        return "Pulling live financial data for `{}`.".format(self.get_input().get("query", "-_-!"))
+        return "Pulling live financial data for `{}`.".format(self.get_input().get("query", "-_-!"))

agent/tools/wikipedia.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 import wikipedia
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class WikipediaParam(ToolParamBase):
@@ -64,19 +64,28 @@ class WikipediaParam(ToolParamBase):
 class Wikipedia(ToolBase, ABC):
     component_name = "Wikipedia"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("Wikipedia processing"):
+            return
+
         if not kwargs.get("query"):
             self.set_output("formalized_content", "")
             return ""
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("Wikipedia processing"):
+                return
+
             try:
                 wikipedia.set_lang(self._param.language)
                 wiki_engine = wikipedia
                 pages = []
                 for p in wiki_engine.search(kwargs["query"], results=self._param.top_n):
+                    if self.check_if_canceled("Wikipedia processing"):
+                        return
+
                     try:
                         pages.append(wikipedia.page(p))
                     except Exception:
@@ -87,6 +96,9 @@ class Wikipedia(ToolBase, ABC):
                                       get_content=lambda r: r.summary)
                 return self.output("formalized_content")
             except Exception as e:
+                if self.check_if_canceled("Wikipedia processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"Wikipedia error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -99,6 +111,6 @@ class Wikipedia(ToolBase, ABC):
 
     def thoughts(self) -> str:
         return """
-Keywords: {} 
+Keywords: {}
 Looking for the most relevant articles.
-        """.format(self.get_input().get("query", "-_-!"))
+        """.format(self.get_input().get("query", "-_-!"))

agent/tools/yahoofinance.py

@@ -20,7 +20,7 @@ from abc import ABC
 import pandas as pd
 import yfinance as yf
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class YahooFinanceParam(ToolParamBase):
@@ -72,17 +72,26 @@ class YahooFinanceParam(ToolParamBase):
 class YahooFinance(ToolBase, ABC):
     component_name = "YahooFinance"
 
-    @timeout(os.environ.get("COMPONENT_EXEC_TIMEOUT", 60))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 60)))
     def _invoke(self, **kwargs):
+        if self.check_if_canceled("YahooFinance processing"):
+            return
+
         if not kwargs.get("stock_code"):
             self.set_output("report", "")
             return ""
 
         last_e = ""
         for _ in range(self._param.max_retries+1):
+            if self.check_if_canceled("YahooFinance processing"):
+                return
+
             yohoo_res = []
             try:
                 msft = yf.Ticker(kwargs["stock_code"])
+                if self.check_if_canceled("YahooFinance processing"):
+                    return
+
                 if self._param.info:
                     yohoo_res.append("# Information:\n" + pd.Series(msft.info).to_markdown() + "\n")
                 if self._param.history:
@@ -100,6 +109,9 @@ class YahooFinance(ToolBase, ABC):
                 self.set_output("report", "\n\n".join(yohoo_res))
                 return self.output("report")
             except Exception as e:
+                if self.check_if_canceled("YahooFinance processing"):
+                    return
+
                 last_e = e
                 logging.exception(f"YahooFinance error: {e}")
                 time.sleep(self._param.delay_after_error)
@@ -111,4 +123,4 @@ class YahooFinance(ToolBase, ABC):
         assert False, self.output()
 
     def thoughts(self) -> str:
-        return "Pulling live financial data for `{}`.".format(self.get_input().get("stock_code", "-_-!"))
+        return "Pulling live financial data for `{}`.".format(self.get_input().get("stock_code", "-_-!"))

api/apps/__init__.py

@@ -24,15 +24,16 @@ from flask_cors import CORS
 from flasgger import Swagger
 from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
 
-from api.db import StatusEnum
+from common.constants import StatusEnum
 from api.db.db_models import close_connection
 from api.db.services import UserService
-from api.utils import CustomJSONEncoder, commands
+from api.utils.json_encode import CustomJSONEncoder
+from api.utils import commands
 
 from flask_mail import Mail
 from flask_session import Session
 from flask_login import LoginManager
-from api import settings
+from common import settings
 from api.utils.api_utils import server_error_response
 from api.constants import API_VERSION
 

api/apps/api_app.py

@@ -21,7 +21,7 @@ from flask import request, Response
 from api.db.services.llm_service import LLMBundle
 from flask_login import login_required, current_user
 
-from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileType, LLMType, ParserType, FileSource
+from api.db import VALID_FILE_TYPES, FileType
 from api.db.db_models import APIToken, Task, File
 from api.db.services import duplicate_name
 from api.db.services.api_service import APITokenService, API4ConversationService
@@ -32,20 +32,21 @@ from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import queue_tasks, TaskService
 from api.db.services.user_service import UserTenantService
-from api import settings
-from api.utils import get_uuid, current_timestamp, datetime_format
+from common.misc_utils import get_uuid
+from common.constants import RetCode, VALID_TASK_STATUS, LLMType, ParserType, FileSource
 from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request, \
     generate_confirmation_token
 
 from api.utils.file_utils import filename_type, thumbnail
 from rag.app.tag import label_question
-from rag.prompts import keyword_extraction
-from rag.utils.storage_factory import STORAGE_IMPL
+from rag.prompts.generator import keyword_extraction
+from common.time_utils import current_timestamp, datetime_format
 
 from api.db.services.canvas_service import UserCanvasService
 from agent.canvas import Canvas
 from functools import partial
 from pathlib import Path
+from common import settings
 
 
 @manager.route('/new_token', methods=['POST'])  # noqa: F821
@@ -58,7 +59,7 @@ def new_token():
             return get_data_error_result(message="Tenant not found!")
 
         tenant_id = tenants[0].tenant_id
-        obj = {"tenant_id": tenant_id, "token": generate_confirmation_token(tenant_id),
+        obj = {"tenant_id": tenant_id, "token": generate_confirmation_token(),
                "create_time": current_timestamp(),
                "create_date": datetime_format(datetime.now()),
                "update_time": None,
@@ -144,7 +145,7 @@ def set_conversation():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
     try:
         if objs[0].source == "agent":
             e, cvs = UserCanvasService.get_by_id(objs[0].dialog_id)
@@ -185,7 +186,7 @@ def completion():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
     req = request.json
     e, conv = API4ConversationService.get_by_id(req["conversation_id"])
     if not e:
@@ -351,7 +352,7 @@ def get_conversation(conversation_id):
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     try:
         e, conv = API4ConversationService.get_by_id(conversation_id)
@@ -361,7 +362,7 @@ def get_conversation(conversation_id):
         conv = conv.to_dict()
         if token != APIToken.query(dialog_id=conv['dialog_id'])[0].token:
             return get_json_result(data=False, message='Authentication error: API key is invalid for this conversation_id!"',
-                                   code=settings.RetCode.AUTHENTICATION_ERROR)
+                                   code=RetCode.AUTHENTICATION_ERROR)
 
         for referenct_i in conv['reference']:
             if referenct_i is None or len(referenct_i) == 0:
@@ -382,7 +383,7 @@ def upload():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     kb_name = request.form.get("kb_name").strip()
     tenant_id = objs[0].tenant_id
@@ -398,12 +399,12 @@ def upload():
 
     if 'file' not in request.files:
         return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+            data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
 
     file = request.files['file']
     if file.filename == '':
         return get_json_result(
-            data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
+            data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR)
 
     root_folder = FileService.get_root_folder(tenant_id)
     pf_id = root_folder["id"]
@@ -426,10 +427,10 @@ def upload():
                 message="This type of file has not been supported yet!")
 
         location = filename
-        while STORAGE_IMPL.obj_exist(kb_id, location):
+        while settings.STORAGE_IMPL.obj_exist(kb_id, location):
             location += "_"
         blob = request.files['file'].read()
-        STORAGE_IMPL.put(kb_id, location, blob)
+        settings.STORAGE_IMPL.put(kb_id, location, blob)
         doc = {
             "id": get_uuid(),
             "kb_id": kb.id,
@@ -465,10 +466,7 @@ def upload():
     if "run" in form_data.keys():
         if request.form.get("run").strip() == "1":
             try:
-                info = {"run": 1, "progress": 0}
-                info["progress_msg"] = ""
-                info["chunk_num"] = 0
-                info["token_num"] = 0
+                info = {"run": 1, "progress": 0, "progress_msg": "", "chunk_num": 0, "token_num": 0}
                 DocumentService.update_by_id(doc["id"], info)
                 # if str(req["run"]) == TaskStatus.CANCEL.value:
                 tenant_id = DocumentService.get_tenant_id(doc["id"])
@@ -495,17 +493,17 @@ def upload_parse():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     if 'file' not in request.files:
         return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+            data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
 
     file_objs = request.files.getlist('file')
     for file_obj in file_objs:
         if file_obj.filename == '':
             return get_json_result(
-                data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
+                data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR)
 
     doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, objs[0].tenant_id)
     return get_json_result(data=doc_ids)
@@ -518,7 +516,7 @@ def list_chunks():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
 
@@ -536,7 +534,7 @@ def list_chunks():
             )
         kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
 
-        res = settings.retrievaler.chunk_list(doc_id, tenant_id, kb_ids)
+        res = settings.retriever.chunk_list(doc_id, tenant_id, kb_ids)
         res = [
             {
                 "content": res_item["content_with_weight"],
@@ -558,7 +556,7 @@ def get_chunk(chunk_id):
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
     try:
         tenant_id = objs[0].tenant_id
         kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
@@ -583,7 +581,7 @@ def list_kb_docs():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
     tenant_id = objs[0].tenant_id
@@ -636,7 +634,7 @@ def docinfos():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
     req = request.json
     doc_ids = req["doc_ids"]
     docs = DocumentService.get_by_ids(doc_ids)
@@ -650,7 +648,7 @@ def document_rm():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     tenant_id = objs[0].tenant_id
     req = request.json
@@ -697,12 +695,12 @@ def document_rm():
             FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
             File2DocumentService.delete_by_document_id(doc_id)
 
-            STORAGE_IMPL.rm(b, n)
+            settings.STORAGE_IMPL.rm(b, n)
         except Exception as e:
             errors += str(e)
 
     if errors:
-        return get_json_result(data=False, message=errors, code=settings.RetCode.SERVER_ERROR)
+        return get_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
 
     return get_json_result(data=True)
 
@@ -717,7 +715,7 @@ def completion_faq():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     e, conv = API4ConversationService.get_by_id(req["conversation_id"])
     if not e:
@@ -725,8 +723,7 @@ def completion_faq():
     if "quote" not in req:
         req["quote"] = True
 
-    msg = []
-    msg.append({"role": "user", "content": req["word"]})
+    msg = [{"role": "user", "content": req["word"]}]
     if not msg[-1].get("id"):
         msg[-1]["id"] = get_uuid()
     message_id = msg[-1]["id"]
@@ -790,7 +787,7 @@ def completion_faq():
                 if ans["reference"]["chunks"][chunk_idx]["img_id"]:
                     try:
                         bkt, nm = ans["reference"]["chunks"][chunk_idx]["img_id"].split("-")
-                        response = STORAGE_IMPL.get(bkt, nm)
+                        response = settings.STORAGE_IMPL.get(bkt, nm)
                         data_type_picture["url"] = base64.b64encode(response).decode('utf-8')
                         data.append(data_type_picture)
                         break
@@ -835,7 +832,7 @@ def completion_faq():
             if ans["reference"]["chunks"][chunk_idx]["img_id"]:
                 try:
                     bkt, nm = ans["reference"]["chunks"][chunk_idx]["img_id"].split("-")
-                    response = STORAGE_IMPL.get(bkt, nm)
+                    response = settings.STORAGE_IMPL.get(bkt, nm)
                     data_type_picture["url"] = base64.b64encode(response).decode('utf-8')
                     data.append(data_type_picture)
                     break
@@ -856,7 +853,7 @@ def retrieval():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
     kb_ids = req.get("kb_id", [])
@@ -867,7 +864,7 @@ def retrieval():
     similarity_threshold = float(req.get("similarity_threshold", 0.2))
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
     top = int(req.get("top_k", 1024))
-    highlight = bool(req.get("highlight", False)) 
+    highlight = bool(req.get("highlight", False))
 
     try:
         kbs = KnowledgebaseService.get_by_ids(kb_ids)
@@ -875,7 +872,7 @@ def retrieval():
         if len(embd_nms) != 1:
             return get_json_result(
                 data=False, message='Knowledge bases use different embedding models or does not exist."',
-                code=settings.RetCode.AUTHENTICATION_ERROR)
+                code=RetCode.AUTHENTICATION_ERROR)
 
         embd_mdl = LLMBundle(kbs[0].tenant_id, LLMType.EMBEDDING, llm_name=kbs[0].embd_id)
         rerank_mdl = None
@@ -884,7 +881,7 @@ def retrieval():
         if req.get("keyword", False):
             chat_mdl = LLMBundle(kbs[0].tenant_id, LLMType.CHAT)
             question += keyword_extraction(chat_mdl, question)
-        ranks = settings.retrievaler.retrieval(question, embd_mdl, kbs[0].tenant_id, kb_ids, page, size,
+        ranks = settings.retriever.retrieval(question, embd_mdl, kbs[0].tenant_id, kb_ids, page, size,
                                                similarity_threshold, vector_similarity_weight, top,
                                                doc_ids, rerank_mdl=rerank_mdl, highlight= highlight,
                                                rank_feature=label_question(question, kbs))
@@ -894,5 +891,5 @@ def retrieval():
     except Exception as e:
         if str(e).find("not_found") > 0:
             return get_json_result(data=False, message='No chunk found! Check the chunk status please!',
-                                   code=settings.RetCode.DATA_ERROR)
+                                   code=RetCode.DATA_ERROR)
         return server_error_response(e)

api/apps/auth/github.py

@@ -34,7 +34,7 @@ class GithubOAuthClient(OAuthClient):
 
     def fetch_user_info(self, access_token, **kwargs):
         """
-        Fetch github user info.
+        Fetch GitHub user info.
         """
         user_info = {}
         try:

api/apps/auth/oidc.py

@@ -43,7 +43,8 @@ class OIDCClient(OAuthClient):
         self.jwks_uri = config['jwks_uri']
 
 
-    def _load_oidc_metadata(self, issuer):
+    @staticmethod
+    def _load_oidc_metadata(issuer):
         """
         Load OIDC metadata from `/.well-known/openid-configuration`.
         """

api/apps/canvas_app.py

@@ -19,6 +19,7 @@ import re
 import sys
 from functools import partial
 
+import flask
 import trio
 from flask import request, Response
 from flask_login import login_required, current_user
@@ -28,32 +29,29 @@ from api.db import CanvasCategory, FileType
 from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService, API4ConversationService
 from api.db.services.document_service import DocumentService
 from api.db.services.file_service import FileService
+from api.db.services.pipeline_operation_log_service import PipelineOperationLogService
+from api.db.services.task_service import queue_dataflow, CANVAS_DEBUG_DOC_ID, TaskService
 from api.db.services.user_service import TenantService
 from api.db.services.user_canvas_version import UserCanvasVersionService
-from api.settings import RetCode
-from api.utils import get_uuid
+from common.constants import RetCode
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result
 from agent.canvas import Canvas
 from peewee import MySQLDatabase, PostgresqlDatabase
-from api.db.db_models import APIToken
+from api.db.db_models import APIToken, Task
 import time
 
 from api.utils.file_utils import filename_type, read_potential_broken_pdf
+from rag.flow.pipeline import Pipeline
+from rag.nlp import search
 from rag.utils.redis_conn import REDIS_CONN
+from common import settings
 
 
 @manager.route('/templates', methods=['GET'])  # noqa: F821
 @login_required
 def templates():
-    return get_json_result(data=[c.to_dict() for c in CanvasTemplateService.query(canvas_category=CanvasCategory.Agent)])
-
-
-@manager.route('/list', methods=['GET'])  # noqa: F821
-@login_required
-def canvas_list():
-    return get_json_result(data=sorted([c.to_dict() for c in \
-                                 UserCanvasService.query(user_id=current_user.id, canvas_category=CanvasCategory.Agent)], key=lambda x: x["update_time"]*-1)
-                           )
+    return get_json_result(data=[c.to_dict() for c in CanvasTemplateService.get_all()])
 
 
 @manager.route('/rm', methods=['POST'])  # noqa: F821
@@ -77,9 +75,10 @@ def save():
     if not isinstance(req["dsl"], str):
         req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
     req["dsl"] = json.loads(req["dsl"])
+    cate = req.get("canvas_category", CanvasCategory.Agent)
     if "id" not in req:
         req["user_id"] = current_user.id
-        if UserCanvasService.query(user_id=current_user.id, title=req["title"].strip(), canvas_category=CanvasCategory.Agent):
+        if UserCanvasService.query(user_id=current_user.id, title=req["title"].strip(), canvas_category=cate):
             return get_data_error_result(message=f"{req['title'].strip()} already exists.")
         req["id"] = get_uuid()
         if not UserCanvasService.save(**req):
@@ -101,7 +100,7 @@ def save():
 def get(canvas_id):
     if not UserCanvasService.accessible(canvas_id, current_user.id):
         return get_data_error_result(message="canvas not found.")
-    e, c = UserCanvasService.get_by_tenant_id(canvas_id)
+    e, c = UserCanvasService.get_by_canvas_id(canvas_id)
     return get_json_result(data=c)
 
 
@@ -148,8 +147,16 @@ def run():
     if not isinstance(cvs.dsl, str):
         cvs.dsl = json.dumps(cvs.dsl, ensure_ascii=False)
 
+    if cvs.canvas_category == CanvasCategory.DataFlow:
+        task_id = get_uuid()
+        Pipeline(cvs.dsl, tenant_id=current_user.id, doc_id=CANVAS_DEBUG_DOC_ID, task_id=task_id, flow_id=req["id"])
+        ok, error_message = queue_dataflow(tenant_id=user_id, flow_id=req["id"], task_id=task_id, file=files[0], priority=0)
+        if not ok:
+            return get_data_error_result(message=error_message)
+        return get_json_result(data={"message_id": task_id})
+
     try:
-        canvas = Canvas(cvs.dsl, current_user.id, req["id"])
+        canvas = Canvas(cvs.dsl, current_user.id)
     except Exception as e:
         return server_error_response(e)
 
@@ -161,8 +168,10 @@ def run():
 
             cvs.dsl = json.loads(str(canvas))
             UserCanvasService.update_by_id(req["id"], cvs.to_dict())
+
         except Exception as e:
             logging.exception(e)
+            canvas.cancel_task()
             yield "data:" + json.dumps({"code": 500, "message": str(e), "data": False}, ensure_ascii=False) + "\n\n"
 
     resp = Response(sse(), mimetype="text/event-stream")
@@ -170,9 +179,48 @@ def run():
     resp.headers.add_header("Connection", "keep-alive")
     resp.headers.add_header("X-Accel-Buffering", "no")
     resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
+    resp.call_on_close(lambda: canvas.cancel_task())
     return resp
 
 
+@manager.route('/rerun', methods=['POST'])  # noqa: F821
+@validate_request("id", "dsl", "component_id")
+@login_required
+def rerun():
+    req = request.json
+    doc = PipelineOperationLogService.get_documents_info(req["id"])
+    if not doc:
+        return get_data_error_result(message="Document not found.")
+    doc = doc[0]
+    if 0 < doc["progress"] < 1:
+        return get_data_error_result(message=f"`{doc['name']}` is processing...")
+
+    if settings.docStoreConn.indexExist(search.index_name(current_user.id), doc["kb_id"]):
+        settings.docStoreConn.delete({"doc_id": doc["id"]}, search.index_name(current_user.id), doc["kb_id"])
+    doc["progress_msg"] = ""
+    doc["chunk_num"] = 0
+    doc["token_num"] = 0
+    DocumentService.clear_chunk_num_when_rerun(doc["id"])
+    DocumentService.update_by_id(id, doc)
+    TaskService.filter_delete([Task.doc_id == id])
+
+    dsl = req["dsl"]
+    dsl["path"] = [req["component_id"]]
+    PipelineOperationLogService.update_by_id(req["id"], {"dsl": dsl})
+    queue_dataflow(tenant_id=current_user.id, flow_id=req["id"], task_id=get_uuid(), doc_id=doc["id"], priority=0, rerun=True)
+    return get_json_result(data=True)
+
+
+@manager.route('/cancel/<task_id>', methods=['PUT'])  # noqa: F821
+@login_required
+def cancel(task_id):
+    try:
+        REDIS_CONN.set(f"{task_id}-cancel", "x")
+    except Exception as e:
+        logging.exception(e)
+    return get_json_result(data=True)
+
+
 @manager.route('/reset', methods=['POST'])  # noqa: F821
 @validate_request("id")
 @login_required
@@ -198,7 +246,7 @@ def reset():
 
 @manager.route("/upload/<canvas_id>", methods=["POST"])  # noqa: F821
 def upload(canvas_id):
-    e, cvs = UserCanvasService.get_by_tenant_id(canvas_id)
+    e, cvs = UserCanvasService.get_by_canvas_id(canvas_id)
     if not e:
         return get_data_error_result(message="canvas not found.")
 
@@ -332,7 +380,7 @@ def test_db_connect():
         if req["db_type"] in ["mysql", "mariadb"]:
             db = MySQLDatabase(req["database"], user=req["username"], host=req["host"], port=req["port"],
                                password=req["password"])
-        elif req["db_type"] == 'postgresql':
+        elif req["db_type"] == 'postgres':
             db = PostgresqlDatabase(req["database"], user=req["username"], host=req["host"], port=req["port"],
                                     password=req["password"])
         elif req["db_type"] == 'mssql':
@@ -348,6 +396,65 @@ def test_db_connect():
             cursor = db.cursor()
             cursor.execute("SELECT 1")
             cursor.close()
+        elif req["db_type"] == 'IBM DB2':
+            import ibm_db
+            conn_str = (
+                f"DATABASE={req['database']};"
+                f"HOSTNAME={req['host']};"
+                f"PORT={req['port']};"
+                f"PROTOCOL=TCPIP;"
+                f"UID={req['username']};"
+                f"PWD={req['password']};"
+            )
+            logging.info(conn_str)
+            conn = ibm_db.connect(conn_str, "", "")
+            stmt = ibm_db.exec_immediate(conn, "SELECT 1 FROM sysibm.sysdummy1")
+            ibm_db.fetch_assoc(stmt)
+            ibm_db.close(conn)
+            return get_json_result(data="Database Connection Successful!")
+        elif req["db_type"] == 'trino':
+            def _parse_catalog_schema(db_name: str):
+                if not db_name:
+                    return None, None
+                if "." in db_name:
+                    catalog_name, schema_name = db_name.split(".", 1)
+                elif "/" in db_name:
+                    catalog_name, schema_name = db_name.split("/", 1)
+                else:
+                    catalog_name, schema_name = db_name, "default"
+                return catalog_name, schema_name
+            try:
+                import trino
+                import os
+                from trino.auth import BasicAuthentication
+            except Exception as e:
+                return server_error_response(f"Missing dependency 'trino'. Please install: pip install trino, detail: {e}")
+
+            catalog, schema = _parse_catalog_schema(req["database"])
+            if not catalog:
+                return server_error_response("For Trino, 'database' must be 'catalog.schema' or at least 'catalog'.")
+
+            http_scheme = "https" if os.environ.get("TRINO_USE_TLS", "0") == "1" else "http"
+
+            auth = None
+            if http_scheme == "https" and req.get("password"):
+                auth = BasicAuthentication(req.get("username") or "ragflow", req["password"])
+
+            conn = trino.dbapi.connect(
+                host=req["host"],
+                port=int(req["port"] or 8080),
+                user=req["username"] or "ragflow",
+                catalog=catalog,
+                schema=schema or "default",
+                http_scheme=http_scheme,
+                auth=auth
+            )
+            cur = conn.cursor()
+            cur.execute("SELECT 1")
+            cur.fetchall()
+            cur.close()
+            conn.close()
+            return get_json_result(data="Database Connection Successful!")
         else:
             return server_error_response("Unsupported database type.")
         if req["db_type"] != 'mssql':
@@ -375,7 +482,6 @@ def getlistversion(canvas_id):
 @login_required
 def getversion( version_id):
     try:
-
         e, version = UserCanvasVersionService.get_by_id(version_id)
         if version:
             return get_json_result(data=version.to_dict())
@@ -383,22 +489,32 @@ def getversion( version_id):
         return get_json_result(data=f"Error getting history file: {e}")
 
 
-@manager.route('/listteam', methods=['GET'])  # noqa: F821
+@manager.route('/list', methods=['GET'])  # noqa: F821
 @login_required
 def list_canvas():
     keywords = request.args.get("keywords", "")
-    page_number = int(request.args.get("page", 1))
-    items_per_page = int(request.args.get("page_size", 150))
+    page_number = int(request.args.get("page", 0))
+    items_per_page = int(request.args.get("page_size", 0))
     orderby = request.args.get("orderby", "create_time")
-    desc = request.args.get("desc", True)
-    try:
+    canvas_category = request.args.get("canvas_category")
+    if request.args.get("desc", "true").lower() == "false":
+        desc = False
+    else:
+        desc = True
+    owner_ids = [id for id in request.args.get("owner_ids", "").strip().split(",") if id]
+    if not owner_ids:
         tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
+        tenants = [m["tenant_id"] for m in tenants]
+        tenants.append(current_user.id)
         canvas, total = UserCanvasService.get_by_tenant_ids(
-            [m["tenant_id"] for m in tenants], current_user.id, page_number,
-            items_per_page, orderby, desc, keywords, canvas_category=CanvasCategory.Agent)
-        return get_json_result(data={"canvas": canvas, "total": total})
-    except Exception as e:
-        return server_error_response(e)
+            tenants, current_user.id, page_number,
+            items_per_page, orderby, desc, keywords, canvas_category)
+    else:
+        tenants = owner_ids
+        canvas, total = UserCanvasService.get_by_tenant_ids(
+            tenants, current_user.id, 0,
+            0, orderby, desc, keywords, canvas_category)
+    return get_json_result(data={"canvas": canvas, "total": total})
 
 
 @manager.route('/setting', methods=['POST'])  # noqa: F821
@@ -432,11 +548,11 @@ def trace():
     cvs_id = request.args.get("canvas_id")
     msg_id = request.args.get("message_id")
     try:
-        bin = REDIS_CONN.get(f"{cvs_id}-{msg_id}-logs")
-        if not bin:
+        binary = REDIS_CONN.get(f"{cvs_id}-{msg_id}-logs")
+        if not binary:
             return get_json_result(data={})
 
-        return get_json_result(data=json.loads(bin.encode("utf-8")))
+        return get_json_result(data=json.loads(binary.encode("utf-8")))
     except Exception as e:
         logging.exception(e)
 
@@ -474,7 +590,7 @@ def sessions(canvas_id):
 @manager.route('/prompts', methods=['GET'])  # noqa: F821
 @login_required
 def prompts():
-    from rag.prompts.prompts import ANALYZE_TASK_SYSTEM, ANALYZE_TASK_USER, NEXT_STEP, REFLECT, CITATION_PROMPT_TEMPLATE
+    from rag.prompts.generator import ANALYZE_TASK_SYSTEM, ANALYZE_TASK_USER, NEXT_STEP, REFLECT, CITATION_PROMPT_TEMPLATE
     return get_json_result(data={
         "task_analysis": ANALYZE_TASK_SYSTEM +"\n\n"+ ANALYZE_TASK_USER,
         "plan_generation": NEXT_STEP,
@@ -483,3 +599,11 @@ def prompts():
         #"context_ranking": RANK_MEMORY,
         "citation_guidelines": CITATION_PROMPT_TEMPLATE
     })
+
+
+@manager.route('/download', methods=['GET'])  # noqa: F821
+def download():
+    id = request.args.get("id")
+    created_by = request.args.get("created_by")
+    blob = FileService.get_blob(created_by, id)
+    return flask.make_response(blob)

api/apps/chunk_app.py

@@ -21,8 +21,6 @@ import xxhash
 from flask import request
 from flask_login import current_user, login_required
 
-from api import settings
-from api.db import LLMType, ParserType
 from api.db.services.dialog_service import meta_filter
 from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
@@ -33,10 +31,10 @@ from api.utils.api_utils import get_data_error_result, get_json_result, server_e
 from rag.app.qa import beAdoc, rmPrefix
 from rag.app.tag import label_question
 from rag.nlp import rag_tokenizer, search
-from rag.prompts import cross_languages, keyword_extraction
-from rag.prompts.prompts import gen_meta_filter
-from rag.settings import PAGERANK_FLD
-from rag.utils import rmSpace
+from rag.prompts.generator import gen_meta_filter, cross_languages, keyword_extraction
+from common.string_utils import remove_redundant_spaces
+from common.constants import RetCode, LLMType, ParserType, PAGERANK_FLD
+from common import settings
 
 
 @manager.route('/list', methods=['POST'])  # noqa: F821
@@ -61,12 +59,12 @@ def list_chunk():
         }
         if "available_int" in req:
             query["available_int"] = int(req["available_int"])
-        sres = settings.retrievaler.search(query, search.index_name(tenant_id), kb_ids, highlight=True)
+        sres = settings.retriever.search(query, search.index_name(tenant_id), kb_ids, highlight=["content_ltks"])
         res = {"total": sres.total, "chunks": [], "doc": doc.to_dict()}
         for id in sres.ids:
             d = {
                 "chunk_id": id,
-                "content_with_weight": rmSpace(sres.highlight[id]) if question and id in sres.highlight else sres.field[
+                "content_with_weight": remove_redundant_spaces(sres.highlight[id]) if question and id in sres.highlight else sres.field[
                     id].get(
                     "content_with_weight", ""),
                 "doc_id": sres.field[id]["doc_id"],
@@ -84,7 +82,7 @@ def list_chunk():
     except Exception as e:
         if str(e).find("not_found") > 0:
             return get_json_result(data=False, message='No chunk found!',
-                                   code=settings.RetCode.DATA_ERROR)
+                                   code=RetCode.DATA_ERROR)
         return server_error_response(e)
 
 
@@ -116,7 +114,7 @@ def get():
     except Exception as e:
         if str(e).find("NotFoundError") >= 0:
             return get_json_result(data=False, message='Chunk not found!',
-                                   code=settings.RetCode.DATA_ERROR)
+                                   code=RetCode.DATA_ERROR)
         return server_error_response(e)
 
 
@@ -201,7 +199,6 @@ def switch():
 @login_required
 @validate_request("chunk_ids", "doc_id")
 def rm():
-    from rag.utils.storage_factory import STORAGE_IMPL
     req = request.json
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
@@ -215,8 +212,8 @@ def rm():
         chunk_number = len(deleted_chunk_ids)
         DocumentService.decrement_chunk_num(doc.id, doc.kb_id, 1, chunk_number, 0)
         for cid in deleted_chunk_ids:
-            if STORAGE_IMPL.obj_exist(doc.kb_id, cid):
-                STORAGE_IMPL.rm(doc.kb_id, cid)
+            if settings.STORAGE_IMPL.obj_exist(doc.kb_id, cid):
+                settings.STORAGE_IMPL.rm(doc.kb_id, cid)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
@@ -293,7 +290,7 @@ def retrieval_test():
         kb_ids = [kb_ids]
     if not kb_ids:
         return get_json_result(data=False, message='Please specify dataset firstly.',
-                               code=settings.RetCode.DATA_ERROR)
+                               code=RetCode.DATA_ERROR)
 
     doc_ids = req.get("doc_ids", [])
     use_kg = req.get("use_kg", False)
@@ -327,7 +324,7 @@ def retrieval_test():
             else:
                 return get_json_result(
                     data=False, message='Only owner of knowledgebase authorized for this operation.',
-                    code=settings.RetCode.OPERATING_ERROR)
+                    code=RetCode.OPERATING_ERROR)
 
         e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
         if not e:
@@ -347,15 +344,16 @@ def retrieval_test():
             question += keyword_extraction(chat_mdl, question)
 
         labels = label_question(question, [kb])
-        ranks = settings.retrievaler.retrieval(question, embd_mdl, tenant_ids, kb_ids, page, size,
+        ranks = settings.retriever.retrieval(question, embd_mdl, tenant_ids, kb_ids, page, size,
                                float(req.get("similarity_threshold", 0.0)),
                                float(req.get("vector_similarity_weight", 0.3)),
                                top,
-                               doc_ids, rerank_mdl=rerank_mdl, highlight=req.get("highlight"),
+                               doc_ids, rerank_mdl=rerank_mdl,
+                                             highlight=req.get("highlight", False),
                                rank_feature=labels
                                )
         if use_kg:
-            ck = settings.kg_retrievaler.retrieval(question,
+            ck = settings.kg_retriever.retrieval(question,
                                                    tenant_ids,
                                                    kb_ids,
                                                    embd_mdl,
@@ -371,7 +369,7 @@ def retrieval_test():
     except Exception as e:
         if str(e).find("not_found") > 0:
             return get_json_result(data=False, message='No chunk found! Check the chunk status please!',
-                                   code=settings.RetCode.DATA_ERROR)
+                                   code=RetCode.DATA_ERROR)
         return server_error_response(e)
 
 
@@ -385,7 +383,7 @@ def knowledge_graph():
         "doc_ids": [doc_id],
         "knowledge_graph_kwd": ["graph", "mind_map"]
     }
-    sres = settings.retrievaler.search(req, search.index_name(tenant_id), kb_ids)
+    sres = settings.retriever.search(req, search.index_name(tenant_id), kb_ids)
     obj = {"graph": {}, "mind_map": {}}
     for id in sres.ids[:2]:
         ty = sres.field[id]["knowledge_graph_kwd"]

api/apps/connector_app.py

@@ -0,0 +1,295 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import json
+import logging
+import time
+import uuid
+from html import escape
+from typing import Any
+
+from flask import make_response, request
+from flask_login import current_user, login_required
+from google_auth_oauthlib.flow import Flow
+
+from api.db import InputType
+from api.db.services.connector_service import ConnectorService, SyncLogsService
+from api.utils.api_utils import get_data_error_result, get_json_result, validate_request
+from common.constants import RetCode, TaskStatus
+from common.data_source.config import GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI, DocumentSource
+from common.data_source.google_util.constant import GOOGLE_DRIVE_WEB_OAUTH_POPUP_TEMPLATE, GOOGLE_SCOPES
+from common.misc_utils import get_uuid
+from rag.utils.redis_conn import REDIS_CONN
+
+
+@manager.route("/set", methods=["POST"])  # noqa: F821
+@login_required
+def set_connector():
+    req = request.json
+    if req.get("id"):
+        conn = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req}
+        ConnectorService.update_by_id(req["id"], conn)
+    else:
+        req["id"] = get_uuid()
+        conn = {
+            "id": req["id"],
+            "tenant_id": current_user.id,
+            "name": req["name"],
+            "source": req["source"],
+            "input_type": InputType.POLL,
+            "config": req["config"],
+            "refresh_freq": int(req.get("refresh_freq", 30)),
+            "prune_freq": int(req.get("prune_freq", 720)),
+            "timeout_secs": int(req.get("timeout_secs", 60 * 29)),
+            "status": TaskStatus.SCHEDULE,
+        }
+        conn["status"] = TaskStatus.SCHEDULE
+        ConnectorService.save(**conn)
+
+    time.sleep(1)
+    e, conn = ConnectorService.get_by_id(req["id"])
+
+    return get_json_result(data=conn.to_dict())
+
+
+@manager.route("/list", methods=["GET"])  # noqa: F821
+@login_required
+def list_connector():
+    return get_json_result(data=ConnectorService.list(current_user.id))
+
+
+@manager.route("/<connector_id>", methods=["GET"])  # noqa: F821
+@login_required
+def get_connector(connector_id):
+    e, conn = ConnectorService.get_by_id(connector_id)
+    if not e:
+        return get_data_error_result(message="Can't find this Connector!")
+    return get_json_result(data=conn.to_dict())
+
+
+@manager.route("/<connector_id>/logs", methods=["GET"])  # noqa: F821
+@login_required
+def list_logs(connector_id):
+    req = request.args.to_dict(flat=True)
+    arr, total = SyncLogsService.list_sync_tasks(connector_id, int(req.get("page", 1)), int(req.get("page_size", 15)))
+    return get_json_result(data={"total": total, "logs": arr})
+
+
+@manager.route("/<connector_id>/resume", methods=["PUT"])  # noqa: F821
+@login_required
+def resume(connector_id):
+    req = request.json
+    if req.get("resume"):
+        ConnectorService.resume(connector_id, TaskStatus.SCHEDULE)
+    else:
+        ConnectorService.resume(connector_id, TaskStatus.CANCEL)
+    return get_json_result(data=True)
+
+
+@manager.route("/<connector_id>/rebuild", methods=["PUT"])  # noqa: F821
+@login_required
+@validate_request("kb_id")
+def rebuild(connector_id):
+    req = request.json
+    err = ConnectorService.rebuild(req["kb_id"], connector_id, current_user.id)
+    if err:
+        return get_json_result(data=False, message=err, code=RetCode.SERVER_ERROR)
+    return get_json_result(data=True)
+
+
+@manager.route("/<connector_id>/rm", methods=["POST"])  # noqa: F821
+@login_required
+def rm_connector(connector_id):
+    ConnectorService.resume(connector_id, TaskStatus.CANCEL)
+    ConnectorService.delete_by_id(connector_id)
+    return get_json_result(data=True)
+
+
+GOOGLE_WEB_FLOW_STATE_PREFIX = "google_drive_web_flow_state"
+GOOGLE_WEB_FLOW_RESULT_PREFIX = "google_drive_web_flow_result"
+WEB_FLOW_TTL_SECS = 15 * 60
+
+
+def _web_state_cache_key(flow_id: str) -> str:
+    return f"{GOOGLE_WEB_FLOW_STATE_PREFIX}:{flow_id}"
+
+
+def _web_result_cache_key(flow_id: str) -> str:
+    return f"{GOOGLE_WEB_FLOW_RESULT_PREFIX}:{flow_id}"
+
+
+def _load_credentials(payload: str | dict[str, Any]) -> dict[str, Any]:
+    if isinstance(payload, dict):
+        return payload
+    try:
+        return json.loads(payload)
+    except json.JSONDecodeError as exc:  # pragma: no cover - defensive
+        raise ValueError("Invalid Google credentials JSON.") from exc
+
+
+def _get_web_client_config(credentials: dict[str, Any]) -> dict[str, Any]:
+    web_section = credentials.get("web")
+    if not isinstance(web_section, dict):
+        raise ValueError("Google OAuth JSON must include a 'web' client configuration to use browser-based authorization.")
+    return {"web": web_section}
+
+
+def _render_web_oauth_popup(flow_id: str, success: bool, message: str):
+    status = "success" if success else "error"
+    auto_close = "window.close();" if success else ""
+    escaped_message = escape(message)
+    payload_json = json.dumps(
+        {
+            "type": "ragflow-google-drive-oauth",
+            "status": status,
+            "flowId": flow_id or "",
+            "message": message,
+        }
+    )
+    html = GOOGLE_DRIVE_WEB_OAUTH_POPUP_TEMPLATE.format(
+        heading="Authorization complete" if success else "Authorization failed",
+        message=escaped_message,
+        payload_json=payload_json,
+        auto_close=auto_close,
+    )
+    response = make_response(html, 200)
+    response.headers["Content-Type"] = "text/html; charset=utf-8"
+    return response
+
+
+@manager.route("/google-drive/oauth/web/start", methods=["POST"])  # noqa: F821
+@login_required
+@validate_request("credentials")
+def start_google_drive_web_oauth():
+    if not GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI:
+        return get_json_result(
+            code=RetCode.SERVER_ERROR,
+            message="Google Drive OAuth redirect URI is not configured on the server.",
+        )
+
+    req = request.json or {}
+    raw_credentials = req.get("credentials", "")
+    try:
+        credentials = _load_credentials(raw_credentials)
+    except ValueError as exc:
+        return get_json_result(code=RetCode.ARGUMENT_ERROR, message=str(exc))
+
+    if credentials.get("refresh_token"):
+        return get_json_result(
+            code=RetCode.ARGUMENT_ERROR,
+            message="Uploaded credentials already include a refresh token.",
+        )
+
+    try:
+        client_config = _get_web_client_config(credentials)
+    except ValueError as exc:
+        return get_json_result(code=RetCode.ARGUMENT_ERROR, message=str(exc))
+
+    flow_id = str(uuid.uuid4())
+    try:
+        flow = Flow.from_client_config(client_config, scopes=GOOGLE_SCOPES[DocumentSource.GOOGLE_DRIVE])
+        flow.redirect_uri = GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI
+        authorization_url, _ = flow.authorization_url(
+            access_type="offline",
+            include_granted_scopes="true",
+            prompt="consent",
+            state=flow_id,
+        )
+    except Exception as exc:  # pragma: no cover - defensive
+        logging.exception("Failed to create Google OAuth flow: %s", exc)
+        return get_json_result(
+            code=RetCode.SERVER_ERROR,
+            message="Failed to initialize Google OAuth flow. Please verify the uploaded client configuration.",
+        )
+
+    cache_payload = {
+        "user_id": current_user.id,
+        "client_config": client_config,
+        "created_at": int(time.time()),
+    }
+    REDIS_CONN.set_obj(_web_state_cache_key(flow_id), cache_payload, WEB_FLOW_TTL_SECS)
+
+    return get_json_result(
+        data={
+            "flow_id": flow_id,
+            "authorization_url": authorization_url,
+            "expires_in": WEB_FLOW_TTL_SECS,
+        }
+    )
+
+
+@manager.route("/google-drive/oauth/web/callback", methods=["GET"])  # noqa: F821
+def google_drive_web_oauth_callback():
+    state_id = request.args.get("state")
+    error = request.args.get("error")
+    error_description = request.args.get("error_description") or error
+
+    if not state_id:
+        return _render_web_oauth_popup("", False, "Missing OAuth state parameter.")
+
+    state_cache = REDIS_CONN.get(_web_state_cache_key(state_id))
+    if not state_cache:
+        return _render_web_oauth_popup(state_id, False, "Authorization session expired. Please restart from the main window.")
+
+    state_obj = json.loads(state_cache)
+    client_config = state_obj.get("client_config")
+    if not client_config:
+        REDIS_CONN.delete(_web_state_cache_key(state_id))
+        return _render_web_oauth_popup(state_id, False, "Authorization session was invalid. Please retry.")
+
+    if error:
+        REDIS_CONN.delete(_web_state_cache_key(state_id))
+        return _render_web_oauth_popup(state_id, False, error_description or "Authorization was cancelled.")
+
+    code = request.args.get("code")
+    if not code:
+        return _render_web_oauth_popup(state_id, False, "Missing authorization code from Google.")
+
+    try:
+        flow = Flow.from_client_config(client_config, scopes=GOOGLE_SCOPES[DocumentSource.GOOGLE_DRIVE])
+        flow.redirect_uri = GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI
+        flow.fetch_token(code=code)
+    except Exception as exc:  # pragma: no cover - defensive
+        logging.exception("Failed to exchange Google OAuth code: %s", exc)
+        REDIS_CONN.delete(_web_state_cache_key(state_id))
+        return _render_web_oauth_popup(state_id, False, "Failed to exchange tokens with Google. Please retry.")
+
+    creds_json = flow.credentials.to_json()
+    result_payload = {
+        "user_id": state_obj.get("user_id"),
+        "credentials": creds_json,
+    }
+    REDIS_CONN.set_obj(_web_result_cache_key(state_id), result_payload, WEB_FLOW_TTL_SECS)
+    REDIS_CONN.delete(_web_state_cache_key(state_id))
+
+    return _render_web_oauth_popup(state_id, True, "Authorization completed successfully.")
+
+
+@manager.route("/google-drive/oauth/web/result", methods=["POST"])  # noqa: F821
+@login_required
+@validate_request("flow_id")
+def poll_google_drive_web_result():
+    req = request.json or {}
+    flow_id = req.get("flow_id")
+    cache_raw = REDIS_CONN.get(_web_result_cache_key(flow_id))
+    if not cache_raw:
+        return get_json_result(code=RetCode.RUNNING, message="Authorization is still pending.")
+
+    result = json.loads(cache_raw)
+    if result.get("user_id") != current_user.id:
+        return get_json_result(code=RetCode.PERMISSION_ERROR, message="You are not allowed to access this authorization result.")
+
+    REDIS_CONN.delete(_web_result_cache_key(flow_id))
+    return get_json_result(data={"credentials": result.get("credentials")})

api/apps/conversation_app.py

@@ -15,12 +15,10 @@
 #
 import json
 import re
-import traceback
+import logging
 from copy import deepcopy
 from flask import Response, request
 from flask_login import current_user, login_required
-from api import settings
-from api.db import LLMType
 from api.db.db_models import APIToken
 from api.db.services.conversation_service import ConversationService, structure_answer
 from api.db.services.dialog_service import DialogService, ask, chat, gen_mindmap
@@ -29,8 +27,9 @@ from api.db.services.search_service import SearchService
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_service import TenantService, UserTenantService
 from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request
-from rag.prompts.prompt_template import load_prompt
-from rag.prompts.prompts import chunks_format
+from rag.prompts.template import load_prompt
+from rag.prompts.generator import chunks_format
+from common.constants import RetCode, LLMType
 
 
 @manager.route("/set", methods=["POST"])  # noqa: F821
@@ -93,7 +92,7 @@ def get():
                 avatar = dialog[0].icon
                 break
         else:
-            return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+            return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=RetCode.OPERATING_ERROR)
 
         for ref in conv.reference:
             if isinstance(ref, list):
@@ -142,7 +141,7 @@ def rm():
                 if DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id):
                     break
             else:
-                return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+                return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=RetCode.OPERATING_ERROR)
             ConversationService.delete_by_id(cid)
         return get_json_result(data=True)
     except Exception as e:
@@ -155,7 +154,7 @@ def list_conversation():
     dialog_id = request.args["dialog_id"]
     try:
         if not DialogService.query(tenant_id=current_user.id, id=dialog_id):
-            return get_json_result(data=False, message="Only owner of dialog authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+            return get_json_result(data=False, message="Only owner of dialog authorized for this operation.", code=RetCode.OPERATING_ERROR)
         convs = ConversationService.query(dialog_id=dialog_id, order_by=ConversationService.model.create_time, reverse=True)
 
         convs = [d.to_dict() for d in convs]
@@ -226,7 +225,7 @@ def completion():
                 if not is_embedded:
                     ConversationService.update_by_id(conv.id, conv.to_dict())
             except Exception as e:
-                traceback.print_exc()
+                logging.exception(e)
                 yield "data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}}, ensure_ascii=False) + "\n\n"
             yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
 

api/apps/dataflow_app.py

@@ -1,353 +0,0 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-import json
-import re
-import sys
-import time
-from functools import partial
-
-import trio
-from flask import request
-from flask_login import current_user, login_required
-
-from agent.canvas import Canvas
-from agent.component import LLM
-from api.db import CanvasCategory, FileType
-from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService
-from api.db.services.document_service import DocumentService
-from api.db.services.file_service import FileService
-from api.db.services.task_service import queue_dataflow
-from api.db.services.user_canvas_version import UserCanvasVersionService
-from api.db.services.user_service import TenantService
-from api.settings import RetCode
-from api.utils import get_uuid
-from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request
-from api.utils.file_utils import filename_type, read_potential_broken_pdf
-from rag.flow.pipeline import Pipeline
-
-
-@manager.route("/templates", methods=["GET"])  # noqa: F821
-@login_required
-def templates():
-    return get_json_result(data=[c.to_dict() for c in CanvasTemplateService.query(canvas_category=CanvasCategory.DataFlow)])
-
-
-@manager.route("/list", methods=["GET"])  # noqa: F821
-@login_required
-def canvas_list():
-    return get_json_result(data=sorted([c.to_dict() for c in UserCanvasService.query(user_id=current_user.id, canvas_category=CanvasCategory.DataFlow)], key=lambda x: x["update_time"] * -1))
-
-
-@manager.route("/rm", methods=["POST"])  # noqa: F821
-@validate_request("canvas_ids")
-@login_required
-def rm():
-    for i in request.json["canvas_ids"]:
-        if not UserCanvasService.accessible(i, current_user.id):
-            return get_json_result(data=False, message="Only owner of canvas authorized for this operation.", code=RetCode.OPERATING_ERROR)
-        UserCanvasService.delete_by_id(i)
-    return get_json_result(data=True)
-
-
-@manager.route("/set", methods=["POST"])  # noqa: F821
-@validate_request("dsl", "title")
-@login_required
-def save():
-    req = request.json
-    if not isinstance(req["dsl"], str):
-        req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
-    req["dsl"] = json.loads(req["dsl"])
-    req["canvas_category"] = CanvasCategory.DataFlow
-    if "id" not in req:
-        req["user_id"] = current_user.id
-        if UserCanvasService.query(user_id=current_user.id, title=req["title"].strip(), canvas_category=CanvasCategory.DataFlow):
-            return get_data_error_result(message=f"{req['title'].strip()} already exists.")
-        req["id"] = get_uuid()
-
-        if not UserCanvasService.save(**req):
-            return get_data_error_result(message="Fail to save canvas.")
-    else:
-        if not UserCanvasService.accessible(req["id"], current_user.id):
-            return get_json_result(data=False, message="Only owner of canvas authorized for this operation.", code=RetCode.OPERATING_ERROR)
-        UserCanvasService.update_by_id(req["id"], req)
-    # save version
-    UserCanvasVersionService.insert(user_canvas_id=req["id"], dsl=req["dsl"], title="{0}_{1}".format(req["title"], time.strftime("%Y_%m_%d_%H_%M_%S")))
-    UserCanvasVersionService.delete_all_versions(req["id"])
-    return get_json_result(data=req)
-
-
-@manager.route("/get/<canvas_id>", methods=["GET"])  # noqa: F821
-@login_required
-def get(canvas_id):
-    if not UserCanvasService.accessible(canvas_id, current_user.id):
-        return get_data_error_result(message="canvas not found.")
-    e, c = UserCanvasService.get_by_tenant_id(canvas_id)
-    return get_json_result(data=c)
-
-
-@manager.route("/run", methods=["POST"])  # noqa: F821
-@validate_request("id")
-@login_required
-def run():
-    req = request.json
-    flow_id = req.get("id", "")
-    doc_id = req.get("doc_id", "")
-    if not all([flow_id, doc_id]):
-        return get_data_error_result(message="id and doc_id are required.")
-
-    if not DocumentService.get_by_id(doc_id):
-        return get_data_error_result(message=f"Document for {doc_id} not found.")
-
-    user_id = req.get("user_id", current_user.id)
-    if not UserCanvasService.accessible(flow_id, current_user.id):
-        return get_json_result(data=False, message="Only owner of canvas authorized for this operation.", code=RetCode.OPERATING_ERROR)
-
-    e, cvs = UserCanvasService.get_by_id(flow_id)
-    if not e:
-        return get_data_error_result(message="canvas not found.")
-
-    if not isinstance(cvs.dsl, str):
-        cvs.dsl = json.dumps(cvs.dsl, ensure_ascii=False)
-
-    task_id = get_uuid()
-
-    ok, error_message = queue_dataflow(dsl=cvs.dsl, tenant_id=user_id, doc_id=doc_id, task_id=task_id, flow_id=flow_id, priority=0)
-    if not ok:
-        return server_error_response(error_message)
-
-    return get_json_result(data={"task_id": task_id, "flow_id": flow_id})
-
-
-@manager.route("/reset", methods=["POST"])  # noqa: F821
-@validate_request("id")
-@login_required
-def reset():
-    req = request.json
-    flow_id = req.get("id", "")
-    if not flow_id:
-        return get_data_error_result(message="id is required.")
-
-    if not UserCanvasService.accessible(flow_id, current_user.id):
-        return get_json_result(data=False, message="Only owner of canvas authorized for this operation.", code=RetCode.OPERATING_ERROR)
-
-    task_id = req.get("task_id", "")
-
-    try:
-        e, user_canvas = UserCanvasService.get_by_id(req["id"])
-        if not e:
-            return get_data_error_result(message="canvas not found.")
-
-        dataflow = Pipeline(dsl=json.dumps(user_canvas.dsl), tenant_id=current_user.id, flow_id=flow_id, task_id=task_id)
-        dataflow.reset()
-        req["dsl"] = json.loads(str(dataflow))
-        UserCanvasService.update_by_id(req["id"], {"dsl": req["dsl"]})
-        return get_json_result(data=req["dsl"])
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/upload/<canvas_id>", methods=["POST"])  # noqa: F821
-def upload(canvas_id):
-    e, cvs = UserCanvasService.get_by_tenant_id(canvas_id)
-    if not e:
-        return get_data_error_result(message="canvas not found.")
-
-    user_id = cvs["user_id"]
-
-    def structured(filename, filetype, blob, content_type):
-        nonlocal user_id
-        if filetype == FileType.PDF.value:
-            blob = read_potential_broken_pdf(blob)
-
-        location = get_uuid()
-        FileService.put_blob(user_id, location, blob)
-
-        return {
-            "id": location,
-            "name": filename,
-            "size": sys.getsizeof(blob),
-            "extension": filename.split(".")[-1].lower(),
-            "mime_type": content_type,
-            "created_by": user_id,
-            "created_at": time.time(),
-            "preview_url": None,
-        }
-
-    if request.args.get("url"):
-        from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CrawlResult, DefaultMarkdownGenerator, PruningContentFilter
-
-        try:
-            url = request.args.get("url")
-            filename = re.sub(r"\?.*", "", url.split("/")[-1])
-
-            async def adownload():
-                browser_config = BrowserConfig(
-                    headless=True,
-                    verbose=False,
-                )
-                async with AsyncWebCrawler(config=browser_config) as crawler:
-                    crawler_config = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(content_filter=PruningContentFilter()), pdf=True, screenshot=False)
-                    result: CrawlResult = await crawler.arun(url=url, config=crawler_config)
-                    return result
-
-            page = trio.run(adownload())
-            if page.pdf:
-                if filename.split(".")[-1].lower() != "pdf":
-                    filename += ".pdf"
-                return get_json_result(data=structured(filename, "pdf", page.pdf, page.response_headers["content-type"]))
-
-            return get_json_result(data=structured(filename, "html", str(page.markdown).encode("utf-8"), page.response_headers["content-type"], user_id))
-
-        except Exception as e:
-            return server_error_response(e)
-
-    file = request.files["file"]
-    try:
-        DocumentService.check_doc_health(user_id, file.filename)
-        return get_json_result(data=structured(file.filename, filename_type(file.filename), file.read(), file.content_type))
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/input_form", methods=["GET"])  # noqa: F821
-@login_required
-def input_form():
-    flow_id = request.args.get("id")
-    cpn_id = request.args.get("component_id")
-    try:
-        e, user_canvas = UserCanvasService.get_by_id(flow_id)
-        if not e:
-            return get_data_error_result(message="canvas not found.")
-        if not UserCanvasService.query(user_id=current_user.id, id=flow_id):
-            return get_json_result(data=False, message="Only owner of canvas authorized for this operation.", code=RetCode.OPERATING_ERROR)
-
-        dataflow = Pipeline(dsl=json.dumps(user_canvas.dsl), tenant_id=current_user.id, flow_id=flow_id, task_id="")
-
-        return get_json_result(data=dataflow.get_component_input_form(cpn_id))
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/debug", methods=["POST"])  # noqa: F821
-@validate_request("id", "component_id", "params")
-@login_required
-def debug():
-    req = request.json
-    if not UserCanvasService.accessible(req["id"], current_user.id):
-        return get_json_result(data=False, message="Only owner of canvas authorized for this operation.", code=RetCode.OPERATING_ERROR)
-    try:
-        e, user_canvas = UserCanvasService.get_by_id(req["id"])
-        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id)
-        canvas.reset()
-        canvas.message_id = get_uuid()
-        component = canvas.get_component(req["component_id"])["obj"]
-        component.reset()
-
-        if isinstance(component, LLM):
-            component.set_debug_inputs(req["params"])
-        component.invoke(**{k: o["value"] for k, o in req["params"].items()})
-        outputs = component.output()
-        for k in outputs.keys():
-            if isinstance(outputs[k], partial):
-                txt = ""
-                for c in outputs[k]():
-                    txt += c
-                outputs[k] = txt
-        return get_json_result(data=outputs)
-    except Exception as e:
-        return server_error_response(e)
-
-
-# api get list version dsl of canvas
-@manager.route("/getlistversion/<canvas_id>", methods=["GET"])  # noqa: F821
-@login_required
-def getlistversion(canvas_id):
-    try:
-        list = sorted([c.to_dict() for c in UserCanvasVersionService.list_by_canvas_id(canvas_id)], key=lambda x: x["update_time"] * -1)
-        return get_json_result(data=list)
-    except Exception as e:
-        return get_data_error_result(message=f"Error getting history files: {e}")
-
-
-# api get version dsl of canvas
-@manager.route("/getversion/<version_id>", methods=["GET"])  # noqa: F821
-@login_required
-def getversion(version_id):
-    try:
-        e, version = UserCanvasVersionService.get_by_id(version_id)
-        if version:
-            return get_json_result(data=version.to_dict())
-    except Exception as e:
-        return get_json_result(data=f"Error getting history file: {e}")
-
-
-@manager.route("/listteam", methods=["GET"])  # noqa: F821
-@login_required
-def list_canvas():
-    keywords = request.args.get("keywords", "")
-    page_number = int(request.args.get("page", 1))
-    items_per_page = int(request.args.get("page_size", 150))
-    orderby = request.args.get("orderby", "create_time")
-    desc = request.args.get("desc", True)
-    try:
-        tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
-        canvas, total = UserCanvasService.get_by_tenant_ids(
-            [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc, keywords, canvas_category=CanvasCategory.DataFlow
-        )
-        return get_json_result(data={"canvas": canvas, "total": total})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/setting", methods=["POST"])  # noqa: F821
-@validate_request("id", "title", "permission")
-@login_required
-def setting():
-    req = request.json
-    req["user_id"] = current_user.id
-
-    if not UserCanvasService.accessible(req["id"], current_user.id):
-        return get_json_result(data=False, message="Only owner of canvas authorized for this operation.", code=RetCode.OPERATING_ERROR)
-
-    e, flow = UserCanvasService.get_by_id(req["id"])
-    if not e:
-        return get_data_error_result(message="canvas not found.")
-    flow = flow.to_dict()
-    flow["title"] = req["title"]
-    for key in ("description", "permission", "avatar"):
-        if value := req.get(key):
-            flow[key] = value
-
-    num = UserCanvasService.update_by_id(req["id"], flow)
-    return get_json_result(data=num)
-
-
-@manager.route("/trace", methods=["GET"])  # noqa: F821
-def trace():
-    dataflow_id = request.args.get("dataflow_id")
-    task_id = request.args.get("task_id")
-    if not all([dataflow_id, task_id]):
-        return get_data_error_result(message="dataflow_id and task_id are required.")
-
-    e, dataflow_canvas = UserCanvasService.get_by_id(dataflow_id)
-    if not e:
-        return get_data_error_result(message="dataflow not found.")
-
-    dsl_str = json.dumps(dataflow_canvas.dsl, ensure_ascii=False)
-    dataflow = Pipeline(dsl=dsl_str, tenant_id=dataflow_canvas.user_id, flow_id=dataflow_id, task_id=task_id)
-    log = dataflow.fetch_logs()
-
-    return get_json_result(data=log)

api/apps/dialog_app.py

@@ -18,13 +18,13 @@ from flask import request
 from flask_login import login_required, current_user
 from api.db.services import duplicate_name
 from api.db.services.dialog_service import DialogService
-from api.db import StatusEnum
+from common.constants import StatusEnum
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import TenantService, UserTenantService
-from api import settings
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
+from common.constants import RetCode
 from api.utils.api_utils import get_json_result
 
 
@@ -219,7 +219,7 @@ def rm():
             else:
                 return get_json_result(
                     data=False, message='Only owner of dialog authorized for this operation.',
-                    code=settings.RetCode.OPERATING_ERROR)
+                    code=RetCode.OPERATING_ERROR)
             dialog_list.append({"id": id,"status":StatusEnum.INVALID.value})
         DialogService.update_many_by_id(dialog_list)
         return get_json_result(data=True)

api/apps/document_app.py

@@ -23,29 +23,31 @@ import flask
 from flask import request
 from flask_login import current_user, login_required
 
-from api import settings
+from api.common.check_team_permission import check_kb_team_permission
 from api.constants import FILE_NAME_LEN_LIMIT, IMG_BASE64_PREFIX
-from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileSource, FileType, ParserType, TaskStatus
-from api.db.db_models import File, Task
+from api.db import VALID_FILE_TYPES, FileType
+from api.db.db_models import Task
 from api.db.services import duplicate_name
 from api.db.services.document_service import DocumentService, doc_upload_and_parse
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.task_service import TaskService, cancel_all_task_of, queue_tasks
+from api.db.services.task_service import TaskService, cancel_all_task_of
 from api.db.services.user_service import UserTenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import (
     get_data_error_result,
     get_json_result,
     server_error_response,
     validate_request,
 )
-from api.utils.file_utils import filename_type, get_project_base_directory, thumbnail
+from api.utils.file_utils import filename_type, thumbnail
+from common.file_utils import get_project_base_directory
+from common.constants import RetCode, VALID_TASK_STATUS, ParserType, TaskStatus
 from api.utils.web_utils import CONTENT_TYPE_MAP, html2pdf, is_valid_url
 from deepdoc.parser.html_parser import RAGFlowHtmlParser
-from rag.nlp import search
-from rag.utils.storage_factory import STORAGE_IMPL
+from rag.nlp import search, rag_tokenizer
+from common import settings
 
 
 @manager.route("/upload", methods=["POST"])  # noqa: F821
@@ -54,27 +56,29 @@ from rag.utils.storage_factory import STORAGE_IMPL
 def upload():
     kb_id = request.form.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     if "file" not in request.files:
-        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
 
     file_objs = request.files.getlist("file")
     for file_obj in file_objs:
         if file_obj.filename == "":
-            return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR)
         if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
-            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
 
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
-    err, files = FileService.upload_document(kb, file_objs, current_user.id)
+    if not check_kb_team_permission(kb, current_user.id):
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
+    err, files = FileService.upload_document(kb, file_objs, current_user.id)
     if err:
-        return get_json_result(data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
+        return get_json_result(data=files, message="\n".join(err), code=RetCode.SERVER_ERROR)
 
     if not files:
-        return get_json_result(data=files, message="There seems to be an issue with your file format. Please verify it is correct and not corrupted.", code=settings.RetCode.DATA_ERROR)
+        return get_json_result(data=files, message="There seems to be an issue with your file format. Please verify it is correct and not corrupted.", code=RetCode.DATA_ERROR)
     files = [f[0] for f in files]  # remove the blob
 
     return get_json_result(data=files)
@@ -86,14 +90,16 @@ def upload():
 def web_crawl():
     kb_id = request.form.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     name = request.form.get("name")
     url = request.form.get("url")
     if not is_valid_url(url):
-        return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
+    if check_kb_team_permission(kb, current_user.id):
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
     blob = html2pdf(url)
     if not blob:
@@ -112,9 +118,9 @@ def web_crawl():
             raise RuntimeError("This type of file has not been supported yet!")
 
         location = filename
-        while STORAGE_IMPL.obj_exist(kb_id, location):
+        while settings.STORAGE_IMPL.obj_exist(kb_id, location):
             location += "_"
-        STORAGE_IMPL.put(kb_id, location, blob)
+        settings.STORAGE_IMPL.put(kb_id, location, blob)
         doc = {
             "id": get_uuid(),
             "kb_id": kb.id,
@@ -150,12 +156,12 @@ def create():
     req = request.json
     kb_id = req["kb_id"]
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
-        return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
 
     if req["name"].strip() == "":
-        return get_json_result(data=False, message="File name can't be empty.", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="File name can't be empty.", code=RetCode.ARGUMENT_ERROR)
     req["name"] = req["name"].strip()
 
     try:
@@ -182,6 +188,7 @@ def create():
                 "id": get_uuid(),
                 "kb_id": kb.id,
                 "parser_id": kb.parser_id,
+                "pipeline_id": kb.pipeline_id,
                 "parser_config": kb.parser_config,
                 "created_by": current_user.id,
                 "type": FileType.VIRTUAL,
@@ -204,13 +211,13 @@ def create():
 def list_docs():
     kb_id = request.args.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     tenants = UserTenantService.query(user_id=current_user.id)
     for tenant in tenants:
         if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
             break
     else:
-        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
     keywords = request.args.get("keywords", "")
 
     page_number = int(request.args.get("page", 0))
@@ -253,6 +260,8 @@ def list_docs():
         for doc_item in docs:
             if doc_item["thumbnail"] and not doc_item["thumbnail"].startswith(IMG_BASE64_PREFIX):
                 doc_item["thumbnail"] = f"/v1/document/image/{kb_id}-{doc_item['thumbnail']}"
+            if doc_item.get("source_type"):
+                doc_item["source_type"] = doc_item["source_type"].split("/")[0]
 
         return get_json_result(data={"total": tol, "docs": docs})
     except Exception as e:
@@ -266,13 +275,13 @@ def get_filter():
 
     kb_id = req.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     tenants = UserTenantService.query(user_id=current_user.id)
     for tenant in tenants:
         if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
             break
     else:
-        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
 
     keywords = req.get("keywords", "")
 
@@ -304,7 +313,7 @@ def docinfos():
     doc_ids = req["doc_ids"]
     for doc_id in doc_ids:
         if not DocumentService.accessible(doc_id, current_user.id):
-            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     docs = DocumentService.get_by_ids(doc_ids)
     return get_json_result(data=list(docs.dicts()))
 
@@ -314,7 +323,7 @@ def docinfos():
 def thumbnails():
     doc_ids = request.args.getlist("doc_ids")
     if not doc_ids:
-        return get_json_result(data=False, message='Lack of "Document ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "Document ID"', code=RetCode.ARGUMENT_ERROR)
 
     try:
         docs = DocumentService.get_thumbnails(doc_ids)
@@ -337,7 +346,7 @@ def change_status():
     status = str(req.get("status", ""))
 
     if status not in ["0", "1"]:
-        return get_json_result(data=False, message='"Status" must be either 0 or 1!', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='"Status" must be either 0 or 1!', code=RetCode.ARGUMENT_ERROR)
 
     result = {}
     for doc_id in doc_ids:
@@ -379,50 +388,12 @@ def rm():
 
     for doc_id in doc_ids:
         if not DocumentService.accessible4deletion(doc_id, current_user.id):
-            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
-    root_folder = FileService.get_root_folder(current_user.id)
-    pf_id = root_folder["id"]
-    FileService.init_knowledgebase_docs(pf_id, current_user.id)
-    errors = ""
-    kb_table_num_map = {}
-    for doc_id in doc_ids:
-        try:
-            e, doc = DocumentService.get_by_id(doc_id)
-            if not e:
-                return get_data_error_result(message="Document not found!")
-            tenant_id = DocumentService.get_tenant_id(doc_id)
-            if not tenant_id:
-                return get_data_error_result(message="Tenant not found!")
-
-            b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
-
-            TaskService.filter_delete([Task.doc_id == doc_id])
-            if not DocumentService.remove_document(doc, tenant_id):
-                return get_data_error_result(message="Database error (Document removal)!")
-
-            f2d = File2DocumentService.get_by_document_id(doc_id)
-            deleted_file_count = 0
-            if f2d:
-                deleted_file_count = FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
-            File2DocumentService.delete_by_document_id(doc_id)
-            if deleted_file_count > 0:
-                STORAGE_IMPL.rm(b, n)
-
-            doc_parser = doc.parser_id
-            if doc_parser == ParserType.TABLE:
-                kb_id = doc.kb_id
-                if kb_id not in kb_table_num_map:
-                    counts = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
-                    kb_table_num_map[kb_id] = counts
-                kb_table_num_map[kb_id] -= 1
-                if kb_table_num_map[kb_id] <= 0:
-                    KnowledgebaseService.delete_field_map(kb_id)
-        except Exception as e:
-            errors += str(e)
+    errors = FileService.delete_docs(doc_ids, current_user.id)
 
     if errors:
-        return get_json_result(data=False, message=errors, code=settings.RetCode.SERVER_ERROR)
+        return get_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
 
     return get_json_result(data=True)
 
@@ -434,7 +405,7 @@ def run():
     req = request.json
     for doc_id in req["doc_ids"]:
         if not DocumentService.accessible(doc_id, current_user.id):
-            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         kb_table_num_map = {}
         for id in req["doc_ids"]:
@@ -467,20 +438,7 @@ def run():
 
             if str(req["run"]) == TaskStatus.RUNNING.value:
                 doc = doc.to_dict()
-                doc["tenant_id"] = tenant_id
-
-                doc_parser = doc.get("parser_id", ParserType.NAIVE)
-                if doc_parser == ParserType.TABLE:
-                    kb_id = doc.get("kb_id")
-                    if not kb_id:
-                        continue
-                    if kb_id not in kb_table_num_map:
-                        count = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
-                        kb_table_num_map[kb_id] = count
-                        if kb_table_num_map[kb_id] <= 0:
-                            KnowledgebaseService.delete_field_map(kb_id)
-                bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
-                queue_tasks(doc, bucket, name, 0)
+                DocumentService.run(tenant_id, doc, kb_table_num_map)
 
         return get_json_result(data=True)
     except Exception as e:
@@ -493,15 +451,15 @@ def run():
 def rename():
     req = request.json
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
             return get_data_error_result(message="Document not found!")
         if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
-            return get_json_result(data=False, message="The extension of file can't be changed", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="The extension of file can't be changed", code=RetCode.ARGUMENT_ERROR)
         if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
-            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
 
         for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
             if d.name == req["name"]:
@@ -515,6 +473,21 @@ def rename():
             e, file = FileService.get_by_id(informs[0].file_id)
             FileService.update_by_id(file.id, {"name": req["name"]})
 
+        tenant_id = DocumentService.get_tenant_id(req["doc_id"])
+        title_tks = rag_tokenizer.tokenize(req["name"])
+        es_body = {
+            "docnm_kwd": req["name"],
+            "title_tks": title_tks,
+            "title_sm_tks": rag_tokenizer.fine_grained_tokenize(title_tks),
+        }
+        if settings.docStoreConn.indexExist(search.index_name(tenant_id), doc.kb_id):
+            settings.docStoreConn.update(
+                {"doc_id": req["doc_id"]},
+                es_body,
+                search.index_name(tenant_id),
+                doc.kb_id,
+            )
+
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
@@ -529,7 +502,7 @@ def get(doc_id):
             return get_data_error_result(message="Document not found!")
 
         b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
-        response = flask.make_response(STORAGE_IMPL.get(b, n))
+        response = flask.make_response(settings.STORAGE_IMPL.get(b, n))
 
         ext = re.search(r"\.([^.]+)$", doc.name.lower())
         ext = ext.group(1) if ext else None
@@ -546,31 +519,22 @@ def get(doc_id):
 
 @manager.route("/change_parser", methods=["POST"])  # noqa: F821
 @login_required
-@validate_request("doc_id", "parser_id")
+@validate_request("doc_id")
 def change_parser():
+
     req = request.json
-
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
-    try:
-        e, doc = DocumentService.get_by_id(req["doc_id"])
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
+
+    e, doc = DocumentService.get_by_id(req["doc_id"])
+    if not e:
+        return get_data_error_result(message="Document not found!")
+
+    def reset_doc():
+        nonlocal doc
+        e = DocumentService.update_by_id(doc.id, {"pipeline_id": req["pipeline_id"], "parser_id": req["parser_id"], "progress": 0, "progress_msg": "", "run": TaskStatus.UNSTART.value})
         if not e:
             return get_data_error_result(message="Document not found!")
-        if doc.parser_id.lower() == req["parser_id"].lower():
-            if "parser_config" in req:
-                if req["parser_config"] == doc.parser_config:
-                    return get_json_result(data=True)
-            else:
-                return get_json_result(data=True)
-
-        if (doc.type == FileType.VISUAL and req["parser_id"] != "picture") or (re.search(r"\.(ppt|pptx|pages)$", doc.name) and req["parser_id"] != "presentation"):
-            return get_data_error_result(message="Not supported yet!")
-
-        e = DocumentService.update_by_id(doc.id, {"parser_id": req["parser_id"], "progress": 0, "progress_msg": "", "run": TaskStatus.UNSTART.value})
-        if not e:
-            return get_data_error_result(message="Document not found!")
-        if "parser_config" in req:
-            DocumentService.update_parser_config(doc.id, req["parser_config"])
         if doc.token_num > 0:
             e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, doc.process_duration * -1)
             if not e:
@@ -581,6 +545,26 @@ def change_parser():
             if settings.docStoreConn.indexExist(search.index_name(tenant_id), doc.kb_id):
                 settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id)
 
+    try:
+        if "pipeline_id" in req and req["pipeline_id"] != "":
+            if doc.pipeline_id == req["pipeline_id"]:
+                return get_json_result(data=True)
+            DocumentService.update_by_id(doc.id, {"pipeline_id": req["pipeline_id"]})
+            reset_doc()
+            return get_json_result(data=True)
+
+        if doc.parser_id.lower() == req["parser_id"].lower():
+            if "parser_config" in req:
+                if req["parser_config"] == doc.parser_config:
+                    return get_json_result(data=True)
+            else:
+                return get_json_result(data=True)
+
+        if (doc.type == FileType.VISUAL and req["parser_id"] != "picture") or (re.search(r"\.(ppt|pptx|pages)$", doc.name) and req["parser_id"] != "presentation"):
+            return get_data_error_result(message="Not supported yet!")
+        if "parser_config" in req:
+            DocumentService.update_parser_config(doc.id, req["parser_config"])
+        reset_doc()
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
@@ -594,7 +578,7 @@ def get_image(image_id):
         if len(arr) != 2:
             return get_data_error_result(message="Image not found.")
         bkt, nm = image_id.split("-")
-        response = flask.make_response(STORAGE_IMPL.get(bkt, nm))
+        response = flask.make_response(settings.STORAGE_IMPL.get(bkt, nm))
         response.headers.set("Content-Type", "image/JPEG")
         return response
     except Exception as e:
@@ -606,12 +590,12 @@ def get_image(image_id):
 @validate_request("conversation_id")
 def upload_and_parse():
     if "file" not in request.files:
-        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
 
     file_objs = request.files.getlist("file")
     for file_obj in file_objs:
         if file_obj.filename == "":
-            return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR)
 
     doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, current_user.id)
 
@@ -624,7 +608,7 @@ def parse():
     url = request.json.get("url") if request.json else ""
     if url:
         if not is_valid_url(url):
-            return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
         download_path = os.path.join(get_project_base_directory(), "logs/downloads")
         os.makedirs(download_path, exist_ok=True)
         from seleniumwire.webdriver import Chrome, ChromeOptions
@@ -657,13 +641,13 @@ def parse():
 
         r = re.search(r"filename=\"([^\"]+)\"", str(res_headers))
         if not r or not r.group(1):
-            return get_json_result(data=False, message="Can't not identify downloaded file", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="Can't not identify downloaded file", code=RetCode.ARGUMENT_ERROR)
         f = File(r.group(1), os.path.join(download_path, r.group(1)))
         txt = FileService.parse_docs([f], current_user.id)
         return get_json_result(data=txt)
 
     if "file" not in request.files:
-        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
 
     file_objs = request.files.getlist("file")
     txt = FileService.parse_docs(file_objs, current_user.id)
@@ -677,18 +661,18 @@ def parse():
 def set_meta():
     req = request.json
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         meta = json.loads(req["meta"])
         if not isinstance(meta, dict):
-            return get_json_result(data=False, message="Only dictionary type supported.", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="Only dictionary type supported.", code=RetCode.ARGUMENT_ERROR)
         for k, v in meta.items():
             if not isinstance(v, str) and not isinstance(v, int) and not isinstance(v, float):
-                return get_json_result(data=False, message=f"The type is not supported: {v}", code=settings.RetCode.ARGUMENT_ERROR)
+                return get_json_result(data=False, message=f"The type is not supported: {v}", code=RetCode.ARGUMENT_ERROR)
     except Exception as e:
-        return get_json_result(data=False, message=f"Json syntax error: {e}", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message=f"Json syntax error: {e}", code=RetCode.ARGUMENT_ERROR)
     if not isinstance(meta, dict):
-        return get_json_result(data=False, message='Meta data should be in Json map format, like {"key": "value"}', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Meta data should be in Json map format, like {"key": "value"}', code=RetCode.ARGUMENT_ERROR)
 
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])