Fix: add fields for logs. (#11039)

### What problem does this PR solve?

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

.github/workflows/release.yml

@@ -16,7 +16,7 @@ concurrency:
 
 jobs:
   release:
-    runs-on: [ "self-hosted", "overseas" ]
+    runs-on: [ "self-hosted", "ragflow-test" ]
     steps:
       - name: Ensure workspace ownership
         run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
@@ -75,62 +75,20 @@ jobs:
           # The body field does not support environment variable substitution directly.
           body_path: release_body.md
 
-      # https://github.com/marketplace/actions/docker-login
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: infiniflow
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      # https://github.com/marketplace/actions/build-and-push-docker-images
-      - name: Build and push full image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          tags: |
-            infiniflow/ragflow:${{ env.RELEASE_TAG }}
-            infiniflow/ragflow:latest-full
-          file: Dockerfile
-          platforms: linux/amd64
-
-      # https://github.com/marketplace/actions/build-and-push-docker-images
-      - name: Build and push slim image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          tags: |
-            infiniflow/ragflow:${{ env.RELEASE_TAG }}-slim
-            infiniflow/ragflow:latest-slim
-          file: Dockerfile
-          build-args: LIGHTEN=1
-          platforms: linux/amd64
-
-      - name: Build ragflow-sdk
+      - name: Build and push ragflow-sdk
         if: startsWith(github.ref, 'refs/tags/v')
         run: |
-          cd sdk/python && \
-          uv build
+          cd sdk/python && uv build && uv publish --token ${{ secrets.PYPI_API_TOKEN }}
 
-      - name: Publish package distributions to PyPI
-        if: startsWith(github.ref, 'refs/tags/v')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: sdk/python/dist/
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          verbose: true
-
-      - name: Build ragflow-cli
+      - name: Build and push ragflow-cli
         if: startsWith(github.ref, 'refs/tags/v')
         run: |
-          cd admin/client && \
-          uv build
+          cd admin/client && uv build && uv publish --token ${{ secrets.PYPI_API_TOKEN }}
 
-      - name: Publish client package distributions to PyPI
-        if: startsWith(github.ref, 'refs/tags/v')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: admin/client/dist/
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          verbose: true
+      - name: Build and push image
+        run: |
+          echo ${{ secrets.DOCKERHUB_TOKEN }} | sudo docker login --username infiniflow --password-stdin
+          sudo docker build --build-arg NEED_MIRROR=1 -t infiniflow/ragflow:${RELEASE_TAG} -f Dockerfile .
+          sudo docker tag infiniflow/ragflow:${RELEASE_TAG} infiniflow/ragflow:latest
+          sudo docker push infiniflow/ragflow:${RELEASE_TAG}
+          sudo docker push infiniflow/ragflow:latest

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ on:
       - '*.md'
       - '*.mdx'
   pull_request:
-    types: [ opened, synchronize, reopened, labeled ]
+    types: [ labeled, synchronize, reopened ]
     paths-ignore:
       - 'docs/**'
       - '*.md'
@@ -29,7 +29,7 @@ jobs:
     # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution
     # https://github.com/orgs/community/discussions/26261
     if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }}
-    runs-on: [ "self-hosted", "debug" ]
+    runs-on: [ "self-hosted", "ragflow-test" ]
     steps:
       # https://github.com/hmarr/debug-action
       #- uses: hmarr/debug-action@v2
@@ -49,20 +49,20 @@ jobs:
       - name: Check workflow duplication
         if: ${{ !cancelled() && !failure() && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci')) }}
         run: |
-          if [[ ${{ github.event_name }} != 'pull_request' ]]; then
+          if [[ "$GITHUB_EVENT_NAME" != "pull_request" && "$GITHUB_EVENT_NAME" != "schedule" ]]; then
             HEAD=$(git rev-parse HEAD)
             # Find a PR that introduced a given commit
             gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}"
             PR_NUMBER=$(gh pr list --search ${HEAD} --state merged --json number --jq .[0].number)
             echo "HEAD=${HEAD}"
             echo "PR_NUMBER=${PR_NUMBER}"
-            if [[ -n ${PR_NUMBER} ]]; then
+            if [[ -n "${PR_NUMBER}" ]]; then
               PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
-              if [[ -f ${PR_SHA_FP} ]]; then
+              if [[ -f "${PR_SHA_FP}" ]]; then
                 read -r PR_SHA PR_RUN_ID < "${PR_SHA_FP}"
                 # Calculate the hash of the current workspace content
                 HEAD_SHA=$(git rev-parse HEAD^{tree})
-                if [[ ${HEAD_SHA} == ${PR_SHA} ]]; then
+                if [[ "${HEAD_SHA}" == "${PR_SHA}" ]]; then
                   echo "Cancel myself since the workspace content hash is the same with PR #${PR_NUMBER} merged. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${PR_RUN_ID} for details."
                   gh run cancel ${GITHUB_RUN_ID}
                   while true; do
@@ -91,122 +91,140 @@ jobs:
           version: ">=0.11.x"
           args: "check"
 
-      - name: Build ragflow:nightly-slim
-        run: |
-          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
-          sudo docker pull ubuntu:22.04
-          sudo DOCKER_BUILDKIT=1 docker build --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-
       - name: Build ragflow:nightly
         run: |
-          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
-
-      - name: Start ragflow:nightly-slim
-        run: |
-          sudo docker compose -f docker/docker-compose.yml down --volumes --remove-orphans
-          echo -e "\nRAGFLOW_IMAGE=infiniflow/ragflow:nightly-slim" >> docker/.env
-          sudo docker compose -f docker/docker-compose.yml up -d
-
-      - name: Stop ragflow:nightly-slim
-        if: always()  # always run this step even if previous steps failed
-        run: |
-          sudo docker compose -f docker/docker-compose.yml down -v
+          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
+          RAGFLOW_IMAGE=infiniflow/ragflow:${GITHUB_RUN_ID}
+          echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> $GITHUB_ENV
+          sudo docker pull ubuntu:22.04
+          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t ${RAGFLOW_IMAGE} .
+          if [[ "$GITHUB_EVENT_NAME" == "schedule" ]]; then
+            export HTTP_API_TEST_LEVEL=p3
+          else
+            export HTTP_API_TEST_LEVEL=p2
+          fi
+          echo "HTTP_API_TEST_LEVEL=${HTTP_API_TEST_LEVEL}" >> $GITHUB_ENV
+          echo "RAGFLOW_CONTAINER=${GITHUB_RUN_ID}-ragflow-cpu-1" >> $GITHUB_ENV
 
       - name: Start ragflow:nightly
         run: |
-          echo -e "\nRAGFLOW_IMAGE=infiniflow/ragflow:nightly" >> docker/.env
-          sudo docker compose -f docker/docker-compose.yml up -d
+          # Determine runner number (default to 1 if not found)
+          RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true)
+          RUNNER_NUM=${RUNNER_NUM:-1}
+
+          # Compute port numbers using bash arithmetic
+          ES_PORT=$((1200 + RUNNER_NUM * 10))
+          OS_PORT=$((1201 + RUNNER_NUM * 10))
+          INFINITY_THRIFT_PORT=$((23817 + RUNNER_NUM * 10))
+          INFINITY_HTTP_PORT=$((23820 + RUNNER_NUM * 10))
+          INFINITY_PSQL_PORT=$((5432 + RUNNER_NUM * 10))
+          MYSQL_PORT=$((5455 + RUNNER_NUM * 10))
+          MINIO_PORT=$((9000 + RUNNER_NUM * 10))
+          MINIO_CONSOLE_PORT=$((9001 + RUNNER_NUM * 10))
+          REDIS_PORT=$((6379 + RUNNER_NUM * 10))
+          TEI_PORT=$((6380 + RUNNER_NUM * 10))
+          KIBANA_PORT=$((6601 + RUNNER_NUM * 10))
+          SVR_HTTP_PORT=$((9380 + RUNNER_NUM * 10))
+          ADMIN_SVR_HTTP_PORT=$((9381 + RUNNER_NUM * 10))
+          SVR_MCP_PORT=$((9382 + RUNNER_NUM * 10))
+          SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + RUNNER_NUM * 10))
+          SVR_WEB_HTTP_PORT=$((80 + RUNNER_NUM * 10))
+          SVR_WEB_HTTPS_PORT=$((443 + RUNNER_NUM * 10))
+
+          # Persist computed ports into docker/.env so docker-compose uses the correct host bindings
+          echo "" >> docker/.env
+          echo -e "ES_PORT=${ES_PORT}" >> docker/.env
+          echo -e "OS_PORT=${OS_PORT}" >> docker/.env
+          echo -e "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}" >> docker/.env
+          echo -e "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}" >> docker/.env
+          echo -e "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}" >> docker/.env
+          echo -e "MYSQL_PORT=${MYSQL_PORT}" >> docker/.env
+          echo -e "MINIO_PORT=${MINIO_PORT}" >> docker/.env
+          echo -e "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}" >> docker/.env
+          echo -e "REDIS_PORT=${REDIS_PORT}" >> docker/.env
+          echo -e "TEI_PORT=${TEI_PORT}" >> docker/.env
+          echo -e "KIBANA_PORT=${KIBANA_PORT}" >> docker/.env
+          echo -e "SVR_HTTP_PORT=${SVR_HTTP_PORT}" >> docker/.env
+          echo -e "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}" >> docker/.env
+          echo -e "SVR_MCP_PORT=${SVR_MCP_PORT}" >> docker/.env
+          echo -e "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}" >> docker/.env
+          echo -e "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}" >> docker/.env
+          echo -e "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}" >> docker/.env
+
+          echo -e "COMPOSE_PROFILES=\${COMPOSE_PROFILES},tei-cpu" >> docker/.env
+          echo -e "TEI_MODEL=BAAI/bge-small-en-v1.5" >> docker/.env
+          echo -e "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> docker/.env
+          echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> $GITHUB_ENV
+
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} up -d
+          uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python
 
       - name: Run sdk tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
-            export HTTP_API_TEST_LEVEL=p3
-          else
-            export HTTP_API_TEST_LEVEL=p2
-          fi
-          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python && uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
+          source .venv/bin/activate && pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
 
       - name: Run frontend api tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && UV_LINK_MODE=copy uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
+          source .venv/bin/activate && pytest -s --tb=short sdk/python/test/test_frontend_api/get_email.py sdk/python/test/test_frontend_api/test_dataset.py
           
       - name: Run http api tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
-            export HTTP_API_TEST_LEVEL=p3
-          else
-            export HTTP_API_TEST_LEVEL=p2
-          fi
-          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
+          source .venv/bin/activate && pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
         run: |
-          sudo docker compose -f docker/docker-compose.yml down -v
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v
 
       - name: Start ragflow:nightly
         run: |
-          sudo DOC_ENGINE=infinity docker compose -f docker/docker-compose.yml up -d
+          sed -i '1i DOC_ENGINE=infinity' docker/.env
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} up -d
 
       - name: Run sdk tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
-            export HTTP_API_TEST_LEVEL=p3
-          else
-            export HTTP_API_TEST_LEVEL=p2
-          fi
-          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python && DOC_ENGINE=infinity uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
+          source .venv/bin/activate && DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
 
       - name: Run frontend api tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          cd sdk/python && UV_LINK_MODE=copy uv sync --python 3.10 --group test --frozen && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
+          source .venv/bin/activate && DOC_ENGINE=infinity pytest -s --tb=short sdk/python/test/test_frontend_api/get_email.py sdk/python/test/test_frontend_api/test_dataset.py
 
       - name: Run http api tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          export HOST_ADDRESS=http://host.docker.internal:9380
-          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
-            export HTTP_API_TEST_LEVEL=p3
-          else
-            export HTTP_API_TEST_LEVEL=p2
-          fi
-          UV_LINK_MODE=copy uv sync --python 3.10 --only-group test --no-default-groups --frozen && DOC_ENGINE=infinity uv run --only-group test --no-default-groups pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
+          source .venv/bin/activate && DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
         run: |
-          sudo DOC_ENGINE=infinity docker compose -f docker/docker-compose.yml down -v
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v
+          sudo docker rmi -f ${RAGFLOW_IMAGE:-NO_IMAGE} || true

CLAUDE.md

@@ -0,0 +1,116 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It's a full-stack application with:
+- Python backend (Flask-based API server)
+- React/TypeScript frontend (built with UmiJS)
+- Microservices architecture with Docker deployment
+- Multiple data stores (MySQL, Elasticsearch/Infinity, Redis, MinIO)
+
+## Architecture
+
+### Backend (`/api/`)
+- **Main Server**: `api/ragflow_server.py` - Flask application entry point
+- **Apps**: Modular Flask blueprints in `api/apps/` for different functionalities:
+  - `kb_app.py` - Knowledge base management
+  - `dialog_app.py` - Chat/conversation handling
+  - `document_app.py` - Document processing
+  - `canvas_app.py` - Agent workflow canvas
+  - `file_app.py` - File upload/management
+- **Services**: Business logic in `api/db/services/`
+- **Models**: Database models in `api/db/db_models.py`
+
+### Core Processing (`/rag/`)
+- **Document Processing**: `deepdoc/` - PDF parsing, OCR, layout analysis
+- **LLM Integration**: `rag/llm/` - Model abstractions for chat, embedding, reranking
+- **RAG Pipeline**: `rag/flow/` - Chunking, parsing, tokenization
+- **Graph RAG**: `graphrag/` - Knowledge graph construction and querying
+
+### Agent System (`/agent/`)
+- **Components**: Modular workflow components (LLM, retrieval, categorize, etc.)
+- **Templates**: Pre-built agent workflows in `agent/templates/`
+- **Tools**: External API integrations (Tavily, Wikipedia, SQL execution, etc.)
+
+### Frontend (`/web/`)
+- React/TypeScript with UmiJS framework
+- Ant Design + shadcn/ui components
+- State management with Zustand
+- Tailwind CSS for styling
+
+## Common Development Commands
+
+### Backend Development
+```bash
+# Install Python dependencies
+uv sync --python 3.10 --all-extras
+uv run download_deps.py
+pre-commit install
+
+# Start dependent services
+docker compose -f docker/docker-compose-base.yml up -d
+
+# Run backend (requires services to be running)
+source .venv/bin/activate
+export PYTHONPATH=$(pwd)
+bash docker/launch_backend_service.sh
+
+# Run tests
+uv run pytest
+
+# Linting
+ruff check
+ruff format
+```
+
+### Frontend Development
+```bash
+cd web
+npm install
+npm run dev        # Development server
+npm run build      # Production build
+npm run lint       # ESLint
+npm run test       # Jest tests
+```
+
+### Docker Development
+```bash
+# Full stack with Docker
+cd docker
+docker compose -f docker-compose.yml up -d
+
+# Check server status
+docker logs -f ragflow-server
+
+# Rebuild images
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
+```
+
+## Key Configuration Files
+
+- `docker/.env` - Environment variables for Docker deployment
+- `docker/service_conf.yaml.template` - Backend service configuration
+- `pyproject.toml` - Python dependencies and project configuration
+- `web/package.json` - Frontend dependencies and scripts
+
+## Testing
+
+- **Python**: pytest with markers (p1/p2/p3 priority levels)
+- **Frontend**: Jest with React Testing Library
+- **API Tests**: HTTP API and SDK tests in `test/` and `sdk/python/test/`
+
+## Database Engines
+
+RAGFlow supports switching between Elasticsearch (default) and Infinity:
+- Set `DOC_ENGINE=infinity` in `docker/.env` to use Infinity
+- Requires container restart: `docker compose down -v && docker compose up -d`
+
+## Development Environment Requirements
+
+- Python 3.10-3.12
+- Node.js >=18.20.4
+- Docker & Docker Compose
+- uv package manager
+- 16GB+ RAM, 50GB+ disk space

Dockerfile

@@ -4,8 +4,6 @@ USER root
 SHELL ["/bin/bash", "-c"]
 
 ARG NEED_MIRROR=0
-ARG LIGHTEN=0
-ENV LIGHTEN=${LIGHTEN}
 
 WORKDIR /ragflow
 
@@ -17,13 +15,6 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co
         /huggingface.co/InfiniFlow/text_concat_xgb_v1.0 \
         /huggingface.co/InfiniFlow/deepdoc \
         | tar -xf - --strip-components=3 -C /ragflow/rag/res/deepdoc 
-RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co,target=/huggingface.co \
-    if [ "$LIGHTEN" != "1" ]; then \
-        (tar -cf - \
-            /huggingface.co/BAAI/bge-large-zh-v1.5 \
-            /huggingface.co/maidalun1020/bce-embedding-base_v1 \
-            | tar -xf - --strip-components=2 -C /root/.ragflow) \
-    fi
 
 # https://github.com/chrismattmann/tika-python
 # This is the only way to run python-tika without internet access. Without this set, the default is to check the tika version and pull latest every time from Apache.
@@ -63,11 +54,11 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     apt install -y ghostscript
 
 RUN if [ "$NEED_MIRROR" == "1" ]; then \
-        pip3 config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
-        pip3 config set global.trusted-host mirrors.aliyun.com; \
+        pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \
+        pip3 config set global.trusted-host pypi.tuna.tsinghua.edu.cn; \
         mkdir -p /etc/uv && \
         echo "[[index]]" > /etc/uv/uv.toml && \
-        echo 'url = "https://mirrors.aliyun.com/pypi/simple"' >> /etc/uv/uv.toml && \
+        echo 'url = "https://pypi.tuna.tsinghua.edu.cn/simple"' >> /etc/uv/uv.toml && \
         echo "default = true" >> /etc/uv/uv.toml; \
     fi; \
     pipx install uv
@@ -151,15 +142,11 @@ COPY pyproject.toml uv.lock ./
 # uv records index url into uv.lock but doesn't failover among multiple indexes
 RUN --mount=type=cache,id=ragflow_uv,target=/root/.cache/uv,sharing=locked \
     if [ "$NEED_MIRROR" == "1" ]; then \
-        sed -i 's|pypi.org|mirrors.aliyun.com/pypi|g' uv.lock; \
+        sed -i 's|pypi.org|pypi.tuna.tsinghua.edu.cn|g' uv.lock; \
     else \
-        sed -i 's|mirrors.aliyun.com/pypi|pypi.org|g' uv.lock; \
+        sed -i 's|pypi.tuna.tsinghua.edu.cn|pypi.org|g' uv.lock; \
     fi; \
-    if [ "$LIGHTEN" == "1" ]; then \
-        uv sync --python 3.10 --frozen; \
-    else \
-        uv sync --python 3.10 --frozen --all-extras; \
-    fi
+    uv sync --python 3.10 --frozen
 
 COPY web web
 COPY docs docs
@@ -169,11 +156,7 @@ RUN --mount=type=cache,id=ragflow_npm,target=/root/.npm,sharing=locked \
 COPY .git /ragflow/.git
 
 RUN version_info=$(git describe --tags --match=v* --first-parent --always); \
-    if [ "$LIGHTEN" == "1" ]; then \
-        version_info="$version_info slim"; \
-    else \
-        version_info="$version_info full"; \
-    fi; \
+    version_info="$version_info"; \
     echo "RAGFlow version: $version_info"; \
     echo $version_info > /ragflow/VERSION
 
@@ -202,6 +185,7 @@ COPY agentic_reasoning agentic_reasoning
 COPY pyproject.toml uv.lock ./
 COPY mcp mcp
 COPY plugin plugin
+COPY common common
 
 COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template
 COPY docker/entrypoint.sh ./

Dockerfile_tei

@@ -0,0 +1,14 @@
+FROM ghcr.io/huggingface/text-embeddings-inference:cpu-1.8
+
+# uv tool install huggingface_hub
+# hf download --local-dir tei_data/BAAI/bge-small-en-v1.5 BAAI/bge-small-en-v1.5
+# hf download --local-dir tei_data/BAAI/bge-m3 BAAI/bge-m3
+# hf download --local-dir tei_data/Qwen/Qwen3-Embedding-0.6B Qwen/Qwen3-Embedding-0.6B
+COPY tei_data /data
+
+# curl -X POST http://localhost:6380/embed -H "Content-Type: application/json" -d '{"inputs": "Hello, world! This is a test sentence."}'
+# curl -X POST http://tei:80/embed -H "Content-Type: application/json" -d '{"inputs": "Hello, world! This is a test sentence."}'
+# [[-0.058816575,0.019564206,0.026697718,...]]
+
+# curl -X POST http://localhost:6380/v1/embeddings -H "Content-Type: application/json" -d '{"input": "Hello, world! This is a test sentence."}'
+# {"object":"list","data":[{"object":"embedding","embedding":[-0.058816575,0.019564206,...],"index":0}],"model":"BAAI/bge-small-en-v1.5","usage":{"prompt_tokens":12,"total_tokens":12}}

README.md

@@ -43,7 +43,9 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
 
 <div align="center">
 <a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
@@ -84,6 +86,7 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Latest Updates
 
+- 2025-10-23 Supports MinerU & Docling as document parsing methods.
 - 2025-10-15 Supports orchestrable ingestion pipeline.
 - 2025-08-08 Supports OpenAI's latest GPT-5 series models.
 - 2025-08-01 Supports agentic workflow and MCP.
@@ -174,41 +177,42 @@ releases! 🌟
    > ```bash
    > vm.max_map_count=262144
    > ```
-
+   >
 2. Clone the repo:
 
    ```bash
    $ git clone https://github.com/infiniflow/ragflow.git
    ```
-
 3. Start up the server using the pre-built Docker images:
 
 > [!CAUTION]
 > All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
 > If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
 
-   > The command below downloads the `v0.21.1-slim` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.21.1-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.21.1` for the full edition `v0.21.1`.
+> The command below downloads the `v0.21.1-slim` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.21.1-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
 
-   ```bash
+```bash
    $ cd ragflow/docker
    # Use CPU for embedding and DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
 
    # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
-   ```
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
+```
 
-   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-   |-------------------|-----------------|-----------------------|--------------------------|
-   | v0.21.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.21.1-slim      | &approx;2       | ❌                   | Stable release            |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                   | _Unstable_ nightly build  |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                    |
+| ----------------- | --------------- | --------------------- | -------------------------- |
+| v0.21.1           | &approx;9       | ✔️                  | Stable release             |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release             |
+| nightly           | &approx;2       | ❌                    | _Unstable_ nightly build |
+
+> Note: Starting with `v0.22.0`, we ship only the slim edition and no longer append the **-slim** suffix to the image tag.
 
 4. Check the server status after having the server up and running:
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _The following output confirms a successful launch of the system:_
@@ -226,14 +230,17 @@ releases! 🌟
 
    > If you skip this confirmation step and directly log in to RAGFlow, your browser may prompt a `network anormal`
    > error because, at that moment, your RAGFlow may not be fully initialized.
-
+   >
 5. In your web browser, enter the IP address of your server and log in to RAGFlow.
+
    > With the default settings, you only need to enter `http://IP_OF_YOUR_MACHINE` (**sans** port number) as the default
    > HTTP serving port `80` can be omitted when using the default configurations.
+   >
 6. In [service_conf.yaml.template](./docker/service_conf.yaml.template), select the desired LLM factory in `user_default_llm` and update
    the `API_KEY` field with the corresponding API key.
 
    > See [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) for more information.
+   >
 
    _The show is on!_
 
@@ -272,7 +279,6 @@ RAGFlow uses Elasticsearch by default for storing full text and vectors. To swit
 > `-v` will delete the docker container volumes, and the existing data will be cleared.
 
 2. Set `DOC_ENGINE` in **docker/.env** to `infinity`.
-
 3. Start the containers:
 
    ```bash
@@ -286,16 +292,6 @@ RAGFlow uses Elasticsearch by default for storing full text and vectors. To swit
 
 This image is approximately 2 GB in size and relies on external LLM and embedding services.
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 Build a Docker image including embedding models
-
-This image is approximately 9 GB in size. As it includes embedding models, it relies on external LLM services only.
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -309,17 +305,15 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    pipx install uv pre-commit
    ```
-
 2. Clone the source code and install Python dependencies:
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```
-
 3. Launch the dependent services (MinIO, Elasticsearch, Redis, and MySQL) using Docker Compose:
 
    ```bash
@@ -331,13 +325,11 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. If you cannot access HuggingFace, set the `HF_ENDPOINT` environment variable to use a mirror site:
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. If your operating system does not have jemalloc, please install it as follows:
 
    ```bash
@@ -350,7 +342,6 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    # macOS
    sudo brew install jemalloc
    ```
-
 6. Launch backend service:
 
    ```bash
@@ -358,14 +349,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. Install frontend dependencies:
 
    ```bash
    cd web
    npm install
    ```
-
 8. Launch frontend service:
 
    ```bash
@@ -375,14 +364,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    _The following output confirms a successful launch of the system:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
-
 9. Stop RAGFlow front-end and back-end service after development is complete:
 
    ```bash
    pkill -f "ragflow_server.py|task_executor.py"
    ```
 
-
 ## 📚 Documentation
 
 - [Quickstart](https://ragflow.io/docs/dev/)

README_id.md

@@ -43,7 +43,13 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
 
 <details open>
 <summary><b>📕 Daftar Isi </b> </summary>
@@ -80,6 +86,7 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Pembaruan Terbaru
 
+- 2025-10-23 Mendukung MinerU & Docling sebagai metode penguraian dokumen.
 - 2025-10-15 Dukungan untuk jalur data yang terorkestrasi.
 - 2025-08-08 Mendukung model seri GPT-5 terbaru dari OpenAI.
 - 2025-08-01 Mendukung alur kerja agen dan MCP.
@@ -168,41 +175,42 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
    > ```bash
    > vm.max_map_count=262144
    > ```
-
+   >
 2. Clone repositori:
 
    ```bash
    $ git clone https://github.com/infiniflow/ragflow.git
    ```
-
 3. Bangun image Docker pre-built dan jalankan server:
 
 > [!CAUTION]
 > Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
 > Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
 
-> Perintah di bawah ini mengunduh edisi v0.21.1-slim dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.21.1-slim, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server. Misalnya, atur RAGFLOW_IMAGE=infiniflow/ragflow:v0.21.1 untuk edisi lengkap v0.21.1.
+> Perintah di bawah ini mengunduh edisi v0.21.1 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.21.1, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
 
 ```bash
-$ cd ragflow/docker
-# Use CPU for embedding and DeepDoc tasks:
-$ docker compose -f docker-compose.yml up -d
+   $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
 
-# To use GPU to accelerate embedding and DeepDoc tasks:
-# docker compose -f docker-compose-gpu.yml up -d
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
 ```
 
-| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-| ----------------- | --------------- | --------------------- | ------------------------ |
-| v0.21.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
-| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
-| nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-| nightly-slim      | &approx;2       | ❌                    | _Unstable_ nightly build |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                    |
+| ----------------- | --------------- | --------------------- | -------------------------- |
+| v0.21.1           | &approx;9       | ✔️                  | Stable release             |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release             |
+| nightly           | &approx;2       | ❌                    | _Unstable_ nightly build |
+
+> Catatan: Mulai dari `v0.22.0`, kami hanya menyediakan edisi slim dan tidak lagi menambahkan akhiran **-slim** pada tag image.
 
 1. Periksa status server setelah server aktif dan berjalan:
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _Output berikut menandakan bahwa sistem berhasil diluncurkan:_
@@ -220,14 +228,17 @@ $ docker compose -f docker-compose.yml up -d
 
    > Jika Anda melewatkan langkah ini dan langsung login ke RAGFlow, browser Anda mungkin menampilkan error `network anormal`
    > karena RAGFlow mungkin belum sepenuhnya siap.
-
+   >
 2. Buka browser web Anda, masukkan alamat IP server Anda, dan login ke RAGFlow.
+
    > Dengan pengaturan default, Anda hanya perlu memasukkan `http://IP_DEVICE_ANDA` (**tanpa** nomor port) karena
    > port HTTP default `80` bisa dihilangkan saat menggunakan konfigurasi default.
+   >
 3. Dalam [service_conf.yaml.template](./docker/service_conf.yaml.template), pilih LLM factory yang diinginkan di `user_default_llm` dan perbarui
    bidang `API_KEY` dengan kunci API yang sesuai.
 
    > Lihat [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) untuk informasi lebih lanjut.
+   >
 
    _Sistem telah siap digunakan!_
 
@@ -253,16 +264,6 @@ Pembaruan konfigurasi ini memerlukan reboot semua kontainer agar efektif:
 
 Image ini berukuran sekitar 2 GB dan bergantung pada aplikasi LLM eksternal dan embedding.
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 Membangun Docker Image Termasuk Model Embedding
-
-Image ini berukuran sekitar 9 GB. Karena sudah termasuk model embedding, ia hanya bergantung pada aplikasi LLM eksternal.
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -276,17 +277,15 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    pipx install uv pre-commit
    ```
-
 2. Clone kode sumber dan instal dependensi Python:
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```
-
 3. Jalankan aplikasi yang diperlukan (MinIO, Elasticsearch, Redis, dan MySQL) menggunakan Docker Compose:
 
    ```bash
@@ -298,13 +297,11 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. Jika Anda tidak dapat mengakses HuggingFace, atur variabel lingkungan `HF_ENDPOINT` untuk menggunakan situs mirror:
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. Jika sistem operasi Anda tidak memiliki jemalloc, instal sebagai berikut:
 
    ```bash
@@ -315,7 +312,6 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    # mac
    sudo brew install jemalloc
    ```
-
 6. Jalankan aplikasi backend:
 
    ```bash
@@ -323,14 +319,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. Instal dependensi frontend:
 
    ```bash
    cd web
    npm install
    ```
-
 8. Jalankan aplikasi frontend:
 
    ```bash
@@ -340,15 +334,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    _Output berikut menandakan bahwa sistem berhasil diluncurkan:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
-
-
 9. Hentikan layanan front-end dan back-end RAGFlow setelah pengembangan selesai:
 
    ```bash
    pkill -f "ragflow_server.py|task_executor.py"
    ```
 
-
 ## 📚 Dokumentasi
 
 - [Quickstart](https://ragflow.io/docs/dev/)

README_ja.md

@@ -43,7 +43,13 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
 
 ## 💡 RAGFlow とは？
 
@@ -60,6 +66,7 @@
 
 ## 🔥 最新情報
 
+- 2025-10-23 ドキュメント解析方法として MinerU と Docling をサポートします。
 - 2025-10-15 オーケストレーションされたデータパイプラインのサポート。
 - 2025-08-08 OpenAI の最新 GPT-5 シリーズモデルをサポートします。
 - 2025-08-01 エージェントワークフローとMCPをサポート。
@@ -147,41 +154,42 @@
    > ```bash
    > vm.max_map_count=262144
    > ```
-
+   >
 2. リポジトリをクローンする:
 
    ```bash
    $ git clone https://github.com/infiniflow/ragflow.git
    ```
-
 3. ビルド済みの Docker イメージをビルドし、サーバーを起動する:
 
 > [!CAUTION]
 > 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
 > ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
 
-   > 以下のコマンドは、RAGFlow Docker イメージの v0.21.1-slim エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.21.1-slim とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。例えば、完全版 v0.21.1 をダウンロードするには、RAGFLOW_IMAGE=infiniflow/ragflow:v0.21.1 と設定します。
+> 以下のコマンドは、RAGFlow Docker イメージの v0.21.1 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.21.1 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
 
-   ```bash
+```bash
    $ cd ragflow/docker
    # Use CPU for embedding and DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
 
    # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
-   ```
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
+```
 
-   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-   | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.21.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                    |
+| ----------------- | --------------- | --------------------- | -------------------------- |
+| v0.21.1           | &approx;9       | ✔️                  | Stable release             |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release             |
+| nightly           | &approx;2       | ❌                    | _Unstable_ nightly build |
+
+> 注意：`v0.22.0` 以降、当プロジェクトでは slim エディションのみを提供し、イメージタグに **-slim** サフィックスを付けなくなりました。
 
 1. サーバーを立ち上げた後、サーバーの状態を確認する:
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _以下の出力は、システムが正常に起動したことを確認するものです:_
@@ -197,12 +205,15 @@
    ```
 
    > もし確認ステップをスキップして直接 RAGFlow にログインした場合、その時点で RAGFlow が完全に初期化されていない可能性があるため、ブラウザーがネットワーク異常エラーを表示するかもしれません。
-
+   >
 2. ウェブブラウザで、プロンプトに従ってサーバーの IP アドレスを入力し、RAGFlow にログインします。
+
    > デフォルトの設定を使用する場合、デフォルトの HTTP サービングポート `80` は省略できるので、与えられたシナリオでは、`http://IP_OF_YOUR_MACHINE`（ポート番号は省略）だけを入力すればよい。
+   >
 3. [service_conf.yaml.template](./docker/service_conf.yaml.template) で、`user_default_llm` で希望の LLM ファクトリを選択し、`API_KEY` フィールドを対応する API キーで更新する。
 
    > 詳しくは [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) を参照してください。
+   >
 
    _これで初期設定完了！ショーの開幕です！_
 
@@ -231,33 +242,27 @@
 RAGFlow はデフォルトで Elasticsearch を使用して全文とベクトルを保存します。［Infinity］に切り替え（https://github.com/infiniflow/infinity/)、次の手順に従います。
 
 1. 実行中のすべてのコンテナを停止するには：
+
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+
    Note: `-v` は docker コンテナのボリュームを削除し、既存のデータをクリアします。
 2. **docker/.env** の「DOC \_ ENGINE」を「infinity」に設定します。
-
 3. 起動コンテナ：
+
    ```bash
    $ docker compose -f docker-compose.yml up -d
    ```
+
    > [!WARNING]
    > Linux/arm64 マシンでの Infinity への切り替えは正式にサポートされていません。
+   >
 
 ## 🔧 ソースコードで Docker イメージを作成（埋め込みモデルなし）
 
 この Docker イメージのサイズは約 1GB で、外部の大モデルと埋め込みサービスに依存しています。
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 ソースコードをコンパイルした Docker イメージ（埋め込みモデルを含む）
-
-この Docker のサイズは約 9GB で、埋め込みモデルを含むため、外部の大モデルサービスのみが必要です。
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -271,17 +276,15 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    pipx install uv pre-commit
    ```
-
 2. ソースコードをクローンし、Python の依存関係をインストールする:
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```
-
 3. Docker Compose を使用して依存サービス（MinIO、Elasticsearch、Redis、MySQL）を起動する:
 
    ```bash
@@ -293,13 +296,11 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. HuggingFace にアクセスできない場合は、`HF_ENDPOINT` 環境変数を設定してミラーサイトを使用してください:
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. オペレーティングシステムにjemallocがない場合は、次のようにインストールします:
 
    ```bash
@@ -310,7 +311,6 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    # mac
    sudo brew install jemalloc
    ```
-
 6. バックエンドサービスを起動する:
 
    ```bash
@@ -318,14 +318,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. フロントエンドの依存関係をインストールする:
 
    ```bash
    cd web
    npm install
    ```
-
 8. フロントエンドサービスを起動する:
 
    ```bash
@@ -335,14 +333,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    _以下の画面で、システムが正常に起動したことを示します:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
-
 9. 開発が完了したら、RAGFlow のフロントエンド サービスとバックエンド サービスを停止します:
 
    ```bash
    pkill -f "ragflow_server.py|task_executor.py"
    ```
 
-
 ## 📚 ドキュメンテーション
 
 - [Quickstart](https://ragflow.io/docs/dev/)

README_ko.md

@@ -43,7 +43,14 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
+
 
 ## 💡 RAGFlow란?
 
@@ -60,6 +67,7 @@
 
 ## 🔥 업데이트
 
+- 2025-10-23 문서 파싱 방법으로 MinerU 및 Docling을 지원합니다.
 - 2025-10-15 조정된 데이터 파이프라인 지원.
 - 2025-08-08 OpenAI의 최신 GPT-5 시리즈 모델을 지원합니다.
 - 2025-08-01 에이전트 워크플로우와 MCP를 지원합니다.
@@ -160,7 +168,7 @@
 > 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
 > ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
 
-   > 아래 명령어는 RAGFlow Docker 이미지의 v0.21.1-slim 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.21.1-slim과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오. 예를 들어, 전체 버전인 v0.21.1을 다운로드하려면 RAGFLOW_IMAGE=infiniflow/ragflow:v0.21.1로 설정합니다.
+   > 아래 명령어는 RAGFlow Docker 이미지의 v0.21.1 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.21.1과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
 
    ```bash
    $ cd ragflow/docker
@@ -168,20 +176,22 @@
    $ docker compose -f docker-compose.yml up -d
 
    # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
    ```
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.21.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.21.1           | &approx;9       | ✔️                    | Stable release           |
    | v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+   | nightly           | &approx;2       | ❌                    | _Unstable_ nightly build |
+
+   > 참고: `v0.22.0`부터는 slim 에디션만 배포하며 이미지 태그에 **-slim** 접미사를 더 이상 붙이지 않습니다.
 
 1. 서버가 시작된 후 서버 상태를 확인하세요:
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _다음 출력 결과로 시스템이 성공적으로 시작되었음을 확인합니다:_
@@ -247,16 +257,6 @@ RAGFlow 는 기본적으로 Elasticsearch 를 사용하여 전체 텍스트 및
 
 이 Docker 이미지의 크기는 약 1GB이며, 외부 대형 모델과 임베딩 서비스에 의존합니다.
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함)
-
-이 Docker의 크기는 약 9GB이며, 이미 임베딩 모델을 포함하고 있으므로 외부 대형 모델 서비스에만 의존하면 됩니다.
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -276,7 +276,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

README_pt_br.md

@@ -43,7 +43,13 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
+
+<div align="center">
+<a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</div>
 
 <details open>
 <summary><b>📕 Índice</b></summary>
@@ -80,7 +86,8 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Últimas Atualizações
 
-- 10-15-2025 Suporte para pipelines de dados orquestrados.
+- 23-10-2025 Suporta MinerU e Docling como métodos de análise de documentos.
+- 15-10-2025 Suporte para pipelines de dados orquestrados.
 - 08-08-2025 Suporta a mais recente série GPT-5 da OpenAI.
 - 01-08-2025 Suporta fluxo de trabalho agente e MCP.
 - 23-05-2025 Adicione o componente executor de código Python/JS ao Agente.
@@ -147,84 +154,86 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ### 🚀 Iniciar o servidor
 
-1.  Certifique-se de que `vm.max_map_count` >= 262144:
+1. Certifique-se de que `vm.max_map_count` >= 262144:
 
-    > Para verificar o valor de `vm.max_map_count`:
-    >
-    > ```bash
-    > $ sysctl vm.max_map_count
-    > ```
-    >
-    > Se necessário, redefina `vm.max_map_count` para um valor de pelo menos 262144:
-    >
-    > ```bash
-    > # Neste caso, defina para 262144:
-    > $ sudo sysctl -w vm.max_map_count=262144
-    > ```
-    >
-    > Essa mudança será resetada após a reinicialização do sistema. Para garantir que a alteração permaneça permanente, adicione ou atualize o valor de `vm.max_map_count` em **/etc/sysctl.conf**:
-    >
-    > ```bash
-    > vm.max_map_count=262144
-    > ```
+   > Para verificar o valor de `vm.max_map_count`:
+   >
+   > ```bash
+   > $ sysctl vm.max_map_count
+   > ```
+   >
+   > Se necessário, redefina `vm.max_map_count` para um valor de pelo menos 262144:
+   >
+   > ```bash
+   > # Neste caso, defina para 262144:
+   > $ sudo sysctl -w vm.max_map_count=262144
+   > ```
+   >
+   > Essa mudança será resetada após a reinicialização do sistema. Para garantir que a alteração permaneça permanente, adicione ou atualize o valor de `vm.max_map_count` em **/etc/sysctl.conf**:
+   >
+   > ```bash
+   > vm.max_map_count=262144
+   > ```
+   >
+2. Clone o repositório:
 
-2.  Clone o repositório:
-
-    ```bash
-    $ git clone https://github.com/infiniflow/ragflow.git
-    ```
-
-3.  Inicie o servidor usando as imagens Docker pré-compiladas:
+   ```bash
+   $ git clone https://github.com/infiniflow/ragflow.git
+   ```
+3. Inicie o servidor usando as imagens Docker pré-compiladas:
 
 > [!CAUTION]
 > Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
 > Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
 
-    > O comando abaixo baixa a edição `v0.21.1-slim` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.21.1-slim`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor. Por exemplo: defina `RAGFLOW_IMAGE=infiniflow/ragflow:v0.21.1` para a edição completa `v0.21.1`.
+    > O comando abaixo baixa a edição`v0.21.1` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.21.1`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
 
-    ```bash
-    $ cd ragflow/docker
-    # Use CPU for embedding and DeepDoc tasks:
-    $ docker compose -f docker-compose.yml up -d
+```bash
+   $ cd ragflow/docker
+   # Use CPU for embedding and DeepDoc tasks:
+   $ docker compose -f docker-compose.yml up -d
 
-    # To use GPU to accelerate embedding and DeepDoc tasks:
-    # docker compose -f docker-compose-gpu.yml up -d
-    ```
+   # To use GPU to accelerate embedding and DeepDoc tasks:
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
+```
 
-    | Tag da imagem RAGFlow | Tamanho da imagem (GB) | Possui modelos de incorporação? | Estável?                 |
-    | --------------------- | ---------------------- | ------------------------------- | ------------------------ |
-    | v0.21.1               | ~9                     | :heavy_check_mark:              | Lançamento estável       |
-    | v0.21.1-slim          | ~2                     | ❌                              | Lançamento estável       |
-    | nightly               | ~9                     | :heavy_check_mark:              | _Instável_ build noturno |
-    | nightly-slim          | ~2                     | ❌                               | _Instável_ build noturno |
+| Tag da imagem RAGFlow | Tamanho da imagem (GB) | Possui modelos de incorporação? | Estável?                      |
+| --------------------- | ---------------------- | --------------------------------- | ------------------------------ |
+| v0.21.1               | &approx;9              | ✔️                              | Lançamento estável           |
+| v0.21.1-slim          | &approx;2              | ❌                                | Lançamento estável           |
+| nightly               | &approx;2              | ❌                                | Construção noturna instável |
 
-4.  Verifique o status do servidor após tê-lo iniciado:
+    > Observação: A partir da`v0.22.0`, distribuímos apenas a edição slim e não adicionamos mais o sufixo **-slim** às tags das imagens.
 
-    ```bash
-    $ docker logs -f ragflow-server
-    ```
+4. Verifique o status do servidor após tê-lo iniciado:
 
-    _O seguinte resultado confirma o lançamento bem-sucedido do sistema:_
+   ```bash
+   $ docker logs -f docker-ragflow-cpu-1
+   ```
 
-    ```bash
-         ____   ___    ______ ______ __
-        / __ \ /   |  / ____// ____// /____  _      __
-       / /_/ // /| | / / __ / /_   / // __ \| | /| / /
-      / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
-     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
+   _O seguinte resultado confirma o lançamento bem-sucedido do sistema:_
 
-     * Rodando em todos os endereços (0.0.0.0)
-    ```
+   ```bash
+        ____   ___    ______ ______ __
+       / __ \ /   |  / ____// ____// /____  _      __
+      / /_/ // /| | / / __ / /_   / // __ \| | /| / /
+     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ /
+    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/
 
-    > Se você pular essa etapa de confirmação e acessar diretamente o RAGFlow, seu navegador pode exibir um erro `network anormal`, pois, nesse momento, seu RAGFlow pode não estar totalmente inicializado.
+    * Rodando em todos os endereços (0.0.0.0)
+   ```
 
-5.  No seu navegador, insira o endereço IP do seu servidor e faça login no RAGFlow.
+   > Se você pular essa etapa de confirmação e acessar diretamente o RAGFlow, seu navegador pode exibir um erro `network anormal`, pois, nesse momento, seu RAGFlow pode não estar totalmente inicializado.
+   >
+5. No seu navegador, insira o endereço IP do seu servidor e faça login no RAGFlow.
 
-    > Com as configurações padrão, você só precisa digitar `http://IP_DO_SEU_MÁQUINA` (**sem** o número da porta), pois a porta HTTP padrão `80` pode ser omitida ao usar as configurações padrão.
+   > Com as configurações padrão, você só precisa digitar `http://IP_DO_SEU_MÁQUINA` (**sem** o número da porta), pois a porta HTTP padrão `80` pode ser omitida ao usar as configurações padrão.
+   >
+6. Em [service_conf.yaml.template](./docker/service_conf.yaml.template), selecione a fábrica LLM desejada em `user_default_llm` e atualize o campo `API_KEY` com a chave de API correspondente.
 
-6.  Em [service_conf.yaml.template](./docker/service_conf.yaml.template), selecione a fábrica LLM desejada em `user_default_llm` e atualize o campo `API_KEY` com a chave de API correspondente.
-
-    > Consulte [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) para mais informações.
+   > Consulte [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) para mais informações.
+   >
 
 _O show está no ar!_
 
@@ -255,9 +264,9 @@ O RAGFlow usa o Elasticsearch por padrão para armazenar texto completo e vetore
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+
    Note: `-v` irá deletar os volumes do contêiner, e os dados existentes serão apagados.
 2. Defina `DOC_ENGINE` no **docker/.env** para `infinity`.
-
 3. Inicie os contêineres:
 
    ```bash
@@ -271,16 +280,6 @@ O RAGFlow usa o Elasticsearch por padrão para armazenar texto completo e vetore
 
 Esta imagem tem cerca de 2 GB de tamanho e depende de serviços externos de LLM e incorporação.
 
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 Criar uma imagem Docker incluindo modelos de incorporação
-
-Esta imagem tem cerca de 9 GB de tamanho. Como inclui modelos de incorporação, depende apenas de serviços externos de LLM.
-
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
@@ -294,17 +293,15 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    pipx install uv pre-commit
    ```
-
 2. Clone o código-fonte e instale as dependências Python:
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # instala os módulos Python dependentes do RAGFlow
+   uv sync --python 3.10 # instala os módulos Python dependentes do RAGFlow
    uv run download_deps.py
    pre-commit install
    ```
-
 3. Inicie os serviços dependentes (MinIO, Elasticsearch, Redis e MySQL) usando Docker Compose:
 
    ```bash
@@ -316,24 +313,21 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. Se não conseguir acessar o HuggingFace, defina a variável de ambiente `HF_ENDPOINT` para usar um site espelho:
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. Se o seu sistema operacional não tiver jemalloc, instale-o da seguinte maneira:
 
-    ```bash
-    # ubuntu
-    sudo apt-get install libjemalloc-dev
-    # centos
-    sudo yum instalar jemalloc
-    # mac
-    sudo brew install jemalloc
-    ```
-
+   ```bash
+   # ubuntu
+   sudo apt-get install libjemalloc-dev
+   # centos
+   sudo yum instalar jemalloc
+   # mac
+   sudo brew install jemalloc
+   ```
 6. Lance o serviço de back-end:
 
    ```bash
@@ -341,14 +335,12 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. Instale as dependências do front-end:
 
    ```bash
    cd web
    npm install
    ```
-
 8. Lance o serviço de front-end:
 
    ```bash
@@ -358,13 +350,11 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    _O seguinte resultado confirma o lançamento bem-sucedido do sistema:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
-
 9. Pare os serviços de front-end e back-end do RAGFlow após a conclusão do desenvolvimento:
 
-    ```bash
-    pkill -f "ragflow_server.py|task_executor.py"
-    ```
-
+   ```bash
+   pkill -f "ragflow_server.py|task_executor.py"
+   ```
 
 ## 📚 Documentação
 

README_tzh.md

@@ -43,7 +43,9 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
 
 <div align="center">
 <a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
@@ -83,6 +85,7 @@
 
 ## 🔥 近期更新
 
+- 2025-10-23 支援 MinerU 和 Docling 作為文件解析方法。
 - 2025-10-15 支援可編排的資料管道。
 - 2025-08-08 支援 OpenAI 最新的 GPT-5 系列模型。
 - 2025-08-01 支援 agentic workflow 和 MCP
@@ -170,47 +173,48 @@
    > ```bash
    > vm.max_map_count=262144
    > ```
-
+   >
 2. 克隆倉庫：
 
    ```bash
    $ git clone https://github.com/infiniflow/ragflow.git
    ```
-
 3. 進入 **docker** 資料夾，利用事先編譯好的 Docker 映像啟動伺服器：
 
 > [!CAUTION]
 > 所有 Docker 映像檔都是為 x86 平台建置的。目前，我們不提供 ARM64 平台的 Docker 映像檔。
 > 如果您使用的是 ARM64 平台，請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
 
-   > 執行以下指令會自動下載 RAGFlow slim Docker 映像 `v0.21.1-slim`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.21.1-slim` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。例如，你可以透過設定 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.21.1` 來下載 RAGFlow 鏡像的 `v0.21.1` 完整發行版。
+> 執行以下指令會自動下載 RAGFlow slim Docker 映像 `v0.21.1`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.21.1` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
 
-   ```bash
+```bash
    $ cd ragflow/docker
    # Use CPU for embedding and DeepDoc tasks:
    $ docker compose -f docker-compose.yml up -d
 
    # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
-   ```
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
+```
 
-   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-   | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.21.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                    |
+| ----------------- | --------------- | --------------------- | -------------------------- |
+| v0.21.1           | &approx;9       | ✔️                  | Stable release             |
+| v0.21.1-slim      | &approx;2       | ❌                    | Stable release             |
+| nightly           | &approx;2       | ❌                    | _Unstable_ nightly build |
 
-   > [!TIP]
-   > 如果你遇到 Docker 映像檔拉不下來的問題，可以在 **docker/.env** 檔案內根據變數 `RAGFLOW_IMAGE` 的註解提示選擇華為雲或阿里雲的對應映像。
-   >
-   > - 華為雲鏡像名：`swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow`
-   > - 阿里雲鏡像名：`registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow`
+> 注意：自 `v0.22.0` 起，我們僅發佈 slim 版本，並且不再在映像標籤後附加 **-slim** 後綴。
+
+> [!TIP]
+> 如果你遇到 Docker 映像檔拉不下來的問題，可以在 **docker/.env** 檔案內根據變數 `RAGFLOW_IMAGE` 的註解提示選擇華為雲或阿里雲的對應映像。
+>
+> - 華為雲鏡像名：`swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow`
+> - 阿里雲鏡像名：`registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow`
 
 4. 伺服器啟動成功後再次確認伺服器狀態：
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _出現以下介面提示說明伺服器啟動成功：_
@@ -226,12 +230,15 @@
    ```
 
    > 如果您跳過這一步驟系統確認步驟就登入 RAGFlow，你的瀏覽器有可能會提示 `network anormal` 或 `網路異常`，因為 RAGFlow 可能並未完全啟動成功。
-
+   >
 5. 在你的瀏覽器中輸入你的伺服器對應的 IP 位址並登入 RAGFlow。
+
    > 上面這個範例中，您只需輸入 http://IP_OF_YOUR_MACHINE 即可：未改動過設定則無需輸入連接埠（預設的 HTTP 服務連接埠 80）。
+   >
 6. 在 [service_conf.yaml.template](./docker/service_conf.yaml.template) 檔案的 `user_default_llm` 欄位設定 LLM factory，並在 `API_KEY` 欄填入和你選擇的大模型相對應的 API key。
 
    > 詳見 [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup)。
+   >
 
    _好戲開始，接著奏樂接著舞！ _
 
@@ -249,7 +256,7 @@
 
 > [./docker/README](./docker/README.md) 解釋了 [service_conf.yaml.template](./docker/service_conf.yaml.template) 用到的環境變數設定和服務配置。
 
-如需更新預設的 HTTP 服務連接埠(80), 可以在[docker-compose.yml](./docker/docker-compose.yml) 檔案中將配置`80:80` 改為`<YOUR_SERVING_PORT>:80` 。
+如需更新預設的 HTTP 服務連接埠(80), 可以在[docker-compose.yml](./docker/docker-compose.yml) 檔案中將配置 `80:80` 改為 `<YOUR_SERVING_PORT>:80` 。
 
 > 所有系統配置都需要透過系統重新啟動生效：
 >
@@ -266,10 +273,9 @@ RAGFlow 預設使用 Elasticsearch 儲存文字和向量資料. 如果要切換
    ```bash
    $ docker compose -f docker/docker-compose.yml down -v
    ```
+
    Note: `-v` 將會刪除 docker 容器的 volumes，已有的資料會被清空。
-
 2. 設定 **docker/.env** 目錄中的 `DOC_ENGINE` 為 `infinity`.
-
 3. 啟動容器:
 
    ```bash
@@ -286,17 +292,7 @@ RAGFlow 預設使用 Elasticsearch 儲存文字和向量資料. 如果要切換
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 原始碼編譯 Docker 映像（包含 embedding 模型）
-
-本 Docker 大小約 9 GB 左右。由於已包含 embedding 模型，所以只需依賴外部的大模型服務即可。
-
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 以原始碼啟動服務
@@ -307,17 +303,15 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    pipx install uv pre-commit
    export UV_INDEX=https://mirrors.aliyun.com/pypi/simple
    ```
-
 2. 下載原始碼並安裝 Python 依賴：
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```
-
 3. 透過 Docker Compose 啟動依賴的服務（MinIO, Elasticsearch, Redis, and MySQL）：
 
    ```bash
@@ -329,13 +323,11 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    ```
    127.0.0.1       es01 infinity mysql minio redis sandbox-executor-manager
    ```
-
 4. 如果無法存取 HuggingFace，可以把環境變數 `HF_ENDPOINT` 設為對應的鏡像網站：
 
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
-
 5. 如果你的操作系统没有 jemalloc，请按照如下方式安装：
 
    ```bash
@@ -346,7 +338,6 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    # mac
    sudo brew install jemalloc
    ```
-
 6. 啟動後端服務：
 
    ```bash
@@ -354,14 +345,12 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
-
 7. 安裝前端依賴：
 
    ```bash
    cd web
    npm install
    ```
-
 8. 啟動前端服務：
 
    ```bash
@@ -371,15 +360,16 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    以下界面說明系統已成功啟動：_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
+
    ```
 
+   ```
 9. 開發完成後停止 RAGFlow 前端和後端服務：
 
    ```bash
    pkill -f "ragflow_server.py|task_executor.py"
    ```
 
-
 ## 📚 技術文檔
 
 - [Quickstart](https://ragflow.io/docs/dev/)

README_zh.md

@@ -43,7 +43,9 @@
   <a href="https://demo.ragflow.io">Demo</a>
 </h4>
 
-#
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/ragflow-octoverse.png" width="1200"/>
+</div>
 
 <div align="center">
 <a href="https://trendshift.io/repositories/9064" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9064" alt="infiniflow%2Fragflow | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
@@ -83,6 +85,7 @@
 
 ## 🔥 近期更新
 
+- 2025-10-23 支持 MinerU 和 Docling 作为文档解析方法。
 - 2025-10-15 支持可编排的数据管道。
 - 2025-08-08 支持 OpenAI 最新的 GPT-5 系列模型。
 - 2025-08-01 支持 agentic workflow 和 MCP。
@@ -183,7 +186,7 @@
 > 请注意，目前官方提供的所有 Docker 镜像均基于 x86 架构构建，并不提供基于 ARM64 的 Docker 镜像。
 > 如果你的操作系统是 ARM64 架构，请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
 
-   > 运行以下命令会自动下载 RAGFlow slim Docker 镜像 `v0.21.1-slim`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.21.1-slim` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。比如，你可以通过设置 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.21.1` 来下载 RAGFlow 镜像的 `v0.21.1` 完整发行版。
+   > 运行以下命令会自动下载 RAGFlow slim Docker 镜像 `v0.21.1`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.21.1` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
 
    ```bash
    $ cd ragflow/docker
@@ -191,15 +194,17 @@
    $ docker compose -f docker-compose.yml up -d
 
    # To use GPU to accelerate embedding and DeepDoc tasks:
-   # docker compose -f docker-compose-gpu.yml up -d
+   # sed -i '1i DEVICE=gpu' .env
+   # docker compose -f docker-compose.yml up -d
    ```
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.21.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.21.1           | &approx;9       | ✔️                    | Stable release           |
    | v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
-   | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
-   | nightly-slim      | &approx;2       | ❌                     | _Unstable_ nightly build |
+   | nightly           | &approx;2       | ❌                    | _Unstable_ nightly build |
+
+   > 注意：从 `v0.22.0` 开始，我们只发布 slim 版本，并且不再在镜像标签后附加 **-slim** 后缀。
 
    > [!TIP]
    > 如果你遇到 Docker 镜像拉不下来的问题，可以在 **docker/.env** 文件内根据变量 `RAGFLOW_IMAGE` 的注释提示选择华为云或者阿里云的相应镜像。
@@ -210,7 +215,7 @@
 4. 服务器启动成功后再次确认服务器状态：
 
    ```bash
-   $ docker logs -f ragflow-server
+   $ docker logs -f docker-ragflow-cpu-1
    ```
 
    _出现以下界面提示说明服务器启动成功：_
@@ -286,17 +291,7 @@ RAGFlow 默认使用 Elasticsearch 存储文本和向量数据. 如果要切换
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-docker build --platform linux/amd64 --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
-```
-
-## 🔧 源码编译 Docker 镜像（包含 embedding 模型）
-
-本 Docker 大小约 9 GB 左右。由于已包含 embedding 模型，所以只需依赖外部的大模型服务即可。
-
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow/
-docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
+docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
 ```
 
 ## 🔨 以源代码启动服务
@@ -313,7 +308,7 @@ docker build --platform linux/amd64 --build-arg NEED_MIRROR=1 -f Dockerfile -t i
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 --all-extras # install RAGFlow dependent python modules
+   uv sync --python 3.10 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

admin/client/admin_client.py

@@ -473,7 +473,7 @@ class AdminCLI(Cmd):
     def parse_connection_args(self, args: List[str]) -> Dict[str, Any]:
         parser = argparse.ArgumentParser(description='Admin CLI Client', add_help=False)
         parser.add_argument('-h', '--host', default='localhost', help='Admin service host')
-        parser.add_argument('-p', '--port', type=int, default=8080, help='Admin service port')
+        parser.add_argument('-p', '--port', type=int, default=9381, help='Admin service port')
         parser.add_argument('-w', '--password', default='admin', type=str, help='Superuser password')
         parser.add_argument('command', nargs='?', help='Single command')
         try:

admin/server/admin_server.py

@@ -23,9 +23,10 @@ import traceback
 from werkzeug.serving import run_simple
 from flask import Flask
 from routes import admin_bp
-from api.utils.log_utils import init_root_logger
-from api.constants import SERVICE_CONF
-from api import settings
+from common.log_utils import init_root_logger
+from common.constants import SERVICE_CONF
+from common.config_utils import show_configs
+from common import settings
 from config import load_configurations, SERVICE_CONFIGS
 from auth import init_default_admin, setup_auth
 from flask_session import Session
@@ -51,6 +52,7 @@ if __name__ == '__main__':
         os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024 * 1024)
     )
     Session(app)
+    show_configs()
     login_manager = LoginManager()
     login_manager.init_app(app)
     settings.init_settings()
@@ -65,7 +67,7 @@ if __name__ == '__main__':
             port=9381,
             application=app,
             threaded=True,
-            use_reloader=True,
+            use_reloader=False,
             use_debugger=True,
         )
     except Exception:

admin/server/auth.py

@@ -23,21 +23,15 @@ from flask import request, jsonify
 from flask_login import current_user, login_user
 from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
 
-from api import settings
 from api.common.exceptions import AdminException, UserNotFoundError
 from api.db.init_data import encode_to_base64
 from api.db.services import UserService
-from api.db import ActiveEnum, StatusEnum
+from common.constants import ActiveEnum, StatusEnum
 from api.utils.crypt import decrypt
-from api.utils import (
-    current_timestamp,
-    datetime_format,
-    get_format_time,
-    get_uuid,
-)
-from api.utils.api_utils import (
-    construct_response,
-)
+from common.misc_utils import get_uuid
+from common.time_utils import current_timestamp, datetime_format, get_format_time
+from common.connection_utils import construct_response
+from common import settings
 
 
 def setup_auth(login_manager):

admin/server/config.py

@@ -21,7 +21,7 @@ from enum import Enum
 
 from pydantic import BaseModel
 from typing import Any
-from api.utils.configs import read_config
+from common.config_utils import read_config
 from urllib.parse import urlparse
 
 

admin/server/services.py

@@ -17,7 +17,7 @@
 
 import re
 from werkzeug.security import check_password_hash
-from api.db import ActiveEnum
+from common.constants import ActiveEnum
 from api.db.services import UserService
 from api.db.joint_services.user_account_service import create_new_user, delete_user_data
 from api.db.services.canvas_service import UserCanvasService

agent/canvas.py

@@ -26,7 +26,7 @@ from typing import Any, Union, Tuple
 from agent.component import component_class
 from agent.component.base import ComponentBase
 from api.db.services.file_service import FileService
-from api.utils import get_uuid, hash_str2int
+from common.misc_utils import get_uuid, hash_str2int
 from rag.prompts.generator import chunks_format
 from rag.utils.redis_conn import REDIS_CONN
 
@@ -153,6 +153,33 @@ class Graph:
     def get_tenant_id(self):
         return self._tenant_id
 
+    def get_value_with_variable(self,value: str) -> Any:
+        pat = re.compile(r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z:0-9_.-]+|sys\.[a-z_]+)\} *\}*")
+        out_parts = []
+        last = 0
+
+        for m in pat.finditer(value):
+            out_parts.append(value[last:m.start()])
+            key = m.group(1)
+            v = self.get_variable_value(key)
+            if v is None:
+                rep = ""
+            elif isinstance(v, partial):
+                buf = []
+                for chunk in v():
+                    buf.append(chunk)
+                rep = "".join(buf)
+            elif isinstance(v, str):
+                rep = v
+            else:
+                rep = json.dumps(v, ensure_ascii=False)
+
+            out_parts.append(rep)
+            last = m.end()
+
+        out_parts.append(value[last:])
+        return("".join(out_parts))
+
     def get_variable_value(self, exp: str) -> Any:
         exp = exp.strip("{").strip("}").strip(" ").strip("{").strip("}")
         if exp.find("@") < 0:
@@ -161,7 +188,32 @@ class Graph:
         cpn = self.get_component(cpn_id)
         if not cpn:
             raise Exception(f"Can't find variable: '{cpn_id}@{var_nm}'")
-        return cpn["obj"].output(var_nm)
+        parts = var_nm.split(".", 1)
+        root_key = parts[0]
+        rest = parts[1] if len(parts) > 1 else ""
+        root_val = cpn["obj"].output(root_key)
+
+        if not rest:
+            return root_val
+        return self.get_variable_param_value(root_val,rest)
+    
+    def get_variable_param_value(self, obj: Any, path: str) -> Any:
+        cur = obj
+        if not path:
+            return cur
+        for key in path.split('.'):
+            if cur is None:
+                return None
+            if isinstance(cur, str):
+                try:
+                    cur = json.loads(cur)
+                except Exception:
+                    return None
+            if isinstance(cur, dict):
+                cur = cur.get(key)
+            else:
+                cur = getattr(cur, key, None)
+        return cur
 
 
 class Canvas(Graph):
@@ -225,6 +277,14 @@ class Canvas(Graph):
         for k, cpn in self.components.items():
             self.components[k]["obj"].reset(True)
 
+        if kwargs.get("webhook_payload"):
+            for k, cpn in self.components.items():
+                if self.components[k]["obj"].component_name.lower() == "webhook":
+                    for kk, vv in kwargs["webhook_payload"].items():
+                        self.components[k]["obj"].set_output(kk, vv)
+
+            self.components[k]["obj"].reset(True)
+
         for k in kwargs.keys():
             if k in ["query", "user_id", "files"] and kwargs[k]:
                 if k == "files":
@@ -256,12 +316,21 @@ class Canvas(Graph):
         def _run_batch(f, t):
             with ThreadPoolExecutor(max_workers=5) as executor:
                 thr = []
-                for i in range(f, t):
+                i = f
+                while i < t:
                     cpn = self.get_component_obj(self.path[i])
                     if cpn.component_name.lower() in ["begin", "userfillup"]:
                         thr.append(executor.submit(cpn.invoke, inputs=kwargs.get("inputs", {})))
+                        i += 1
                     else:
-                        thr.append(executor.submit(cpn.invoke, **cpn.get_input()))
+                        for _, ele in cpn.get_input_elements().items():
+                            if isinstance(ele, dict) and ele.get("_cpn_id") and ele.get("_cpn_id") not in self.path[:i]:
+                                self.path.pop(i)
+                                t -= 1
+                                break
+                        else:
+                            thr.append(executor.submit(cpn.invoke, **cpn.get_input()))
+                            i += 1
                 for t in thr:
                     t.result()
 
@@ -291,6 +360,7 @@ class Canvas(Graph):
                     "thoughts": self.get_component_thoughts(self.path[i])
                 })
             _run_batch(idx, to)
+            to = len(self.path)
             # post processing of components invocation
             for i in range(idx, to):
                 cpn = self.get_component(self.path[i])

agent/component/agent_with_tools.py

@@ -27,7 +27,7 @@ from agent.tools.base import LLMToolPluginCallSession, ToolParamBase, ToolBase,
 from api.db.services.llm_service import LLMBundle
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.mcp_server_service import MCPServerService
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 from rag.prompts.generator import next_step, COMPLETE_TASK, analyze_task, \
     citation_prompt, reflect, rank_memories, kb_prompt, citation_plus, full_question, message_fit_in
 from rag.utils.mcp_tool_call_conn import MCPToolCallSession, mcp_tool_metadata_to_openai_tool
@@ -158,7 +158,12 @@ class Agent(LLM, ToolBase):
 
         downstreams = self._canvas.get_component(self._id)["downstream"] if self._canvas.get_component(self._id) else []
         ex = self.exception_handler()
-        if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not self._param.output_structure and not (ex and ex["goto"]):
+        output_structure=None
+        try:
+            output_structure=self._param.outputs['structured']
+        except Exception:
+            pass
+        if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not output_structure and not (ex and ex["goto"]):
             self.set_output("content", partial(self.stream_output_with_tools, prompt, msg, user_defined_prompt))
             return
 

agent/component/base.py

@@ -25,7 +25,7 @@ from typing import Any, List, Union
 import pandas as pd
 import trio
 from agent import settings
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 _FEEDED_DEPRECATED_PARAMS = "_feeded_deprecated_params"

agent/component/categorize.py

@@ -18,10 +18,10 @@ import os
 import re
 from abc import ABC
 
-from api.db import LLMType
+from common.constants import LLMType
 from api.db.services.llm_service import LLMBundle
 from agent.component.llm import LLMParam, LLM
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 from rag.llm.chat_model import ERROR_PREFIX
 
 

agent/component/data_operations.py

@@ -0,0 +1,202 @@
+from abc import ABC
+import ast
+import os
+from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.api_utils import timeout
+
+class DataOperationsParam(ComponentParamBase):
+    """
+    Define the Data Operations component parameters.
+    """
+    def __init__(self):
+        super().__init__()
+        self.query = []
+        self.operations = "literal_eval"
+        self.select_keys = []
+        self.filter_values=[]
+        self.updates=[]
+        self.remove_keys=[]
+        self.rename_keys=[]
+        self.outputs = {
+            "result": {
+                "value": [],
+                "type": "Array of Object"
+            }
+        }
+    
+    def check(self):
+        self.check_valid_value(self.operations, "Support operations", ["select_keys", "literal_eval","combine","filter_values","append_or_update","remove_keys","rename_keys"])
+    
+    
+
+class DataOperations(ComponentBase,ABC):
+    component_name = "DataOperations"
+
+    def get_input_form(self) -> dict[str, dict]:
+        return {
+            k: {"name": o.get("name", ""), "type": "line"}
+            for input_item in (self._param.query or [])
+            for k, o in self.get_input_elements_from_text(input_item).items()
+        }
+
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
+    def _invoke(self, **kwargs):
+        self.input_objects=[]
+        inputs = getattr(self._param, "query", None)
+        if not isinstance(inputs, (list, tuple)):
+            inputs = [inputs]
+        for input_ref in inputs:
+            input_object=self._canvas.get_variable_value(input_ref)
+            if input_object is None:
+                continue
+            if isinstance(input_object,dict):
+                self.input_objects.append(input_object)
+            elif isinstance(input_object,list):
+                self.input_objects.extend(x for x in input_object if isinstance(x, dict))
+            else:
+                continue
+        if self._param.operations == "select_keys":
+            self._select_keys()
+        elif self._param.operations == "recursive_eval":
+            self._literal_eval()
+        elif self._param.operations == "combine":
+            self._combine()
+        elif self._param.operations == "filter_values":
+            self._filter_values()
+        elif self._param.operations == "append_or_update":
+            self._append_or_update()
+        elif self._param.operations == "remove_keys":
+            self._remove_keys()
+        else:
+            self._rename_keys()
+    
+    def _select_keys(self):
+        filter_criteria: list[str] = self._param.select_keys
+        results = [{key: value for key, value in data_dict.items() if key in filter_criteria} for data_dict in self.input_objects]
+        self.set_output("result", results)
+
+
+    def _recursive_eval(self, data):
+        if isinstance(data, dict):
+            return {k: self.recursive_eval(v) for k, v in data.items()}
+        if isinstance(data, list):
+            return [self.recursive_eval(item) for item in data]
+        if isinstance(data, str):
+            try:
+                if (
+                    data.strip().startswith(("{", "[", "(", "'", '"'))
+                    or data.strip().lower() in ("true", "false", "none")
+                    or data.strip().replace(".", "").isdigit()
+                ):
+                    return ast.literal_eval(data)
+            except (ValueError, SyntaxError, TypeError, MemoryError):
+                return data
+            else:
+                return data
+        return data
+    
+    def _literal_eval(self):
+        self.set_output("result", self._recursive_eval(self.input_objects))
+
+    def _combine(self):
+        result={}
+        for obj in self.input_objects:
+            for key, value in obj.items():
+                if key not in result:
+                    result[key] = value
+                elif isinstance(result[key], list):
+                    if isinstance(value, list):
+                        result[key].extend(value)
+                    else:
+                        result[key].append(value)
+                else:
+                    result[key] = (
+                        [result[key], value] if not isinstance(value, list) else [result[key], *value]
+                    )
+        self.set_output("result", result)
+    
+    def norm(self,v):
+        s = "" if v is None else str(v)
+        return s
+    
+    def match_rule(self, obj, rule):
+        key = rule.get("key")
+        op = (rule.get("operator") or "equals").lower()
+        target = self.norm(rule.get("value"))
+        target = self._canvas.get_value_with_variable(target) or target
+        if key not in obj:
+            return False
+        val = obj.get(key, None)
+        v = self.norm(val)
+        if op == "=":
+            return v == target
+        if op == "≠":
+            return v != target
+        if op == "contains":
+            return target in v
+        if op == "start with":
+            return v.startswith(target)
+        if op == "end with":
+            return v.endswith(target)
+        return False
+        
+    def _filter_values(self):
+        results=[]
+        rules = (getattr(self._param, "filter_values", None) or [])
+        for obj in self.input_objects:
+            if not rules:
+                results.append(obj)
+                continue
+            if all(self.match_rule(obj, r) for r in rules):
+                results.append(obj)
+        self.set_output("result", results)
+            
+                
+    def _append_or_update(self):
+        results=[]
+        updates = getattr(self._param, "updates", []) or [] 
+        for obj in self.input_objects:
+            new_obj = dict(obj)
+            for item in updates:
+                if not isinstance(item, dict):
+                    continue
+                k = (item.get("key") or "").strip()
+                if not k:
+                    continue
+                new_obj[k] = self._canvas.get_value_with_variable(item.get("value")) or item.get("value")
+            results.append(new_obj)
+        self.set_output("result", results)
+
+    def _remove_keys(self):
+        results = []
+        remove_keys = getattr(self._param, "remove_keys", []) or []
+
+        for obj in (self.input_objects or []):
+            new_obj = dict(obj)
+            for k in remove_keys:
+                if not isinstance(k, str):
+                    continue
+                new_obj.pop(k, None)
+            results.append(new_obj)
+        self.set_output("result", results)
+
+    def _rename_keys(self):
+        results = []
+        rename_pairs = getattr(self._param, "rename_keys", []) or []
+
+        for obj in (self.input_objects or []):
+            new_obj = dict(obj)
+            for pair in rename_pairs:
+                if not isinstance(pair, dict):
+                    continue
+                old = (pair.get("old_key") or "").strip()
+                new = (pair.get("new_key") or "").strip()
+                if not old or not new or old == new:
+                    continue
+                if old in new_obj:
+                    new_obj[new] = new_obj.pop(old)
+            results.append(new_obj)
+        self.set_output("result", results)
+
+    def thoughts(self) -> str:
+        return "DataOperation in progress"

agent/component/invoke.py

@@ -23,7 +23,7 @@ from abc import ABC
 import requests
 
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 from deepdoc.parser import HtmlParser
 
 

agent/component/iteration.py

@@ -16,6 +16,13 @@
 from abc import ABC
 from agent.component.base import ComponentBase, ComponentParamBase
 
+"""
+class VariableModel(BaseModel):
+    data_type: Annotated[Literal["string", "number", "Object", "Boolean", "Array<string>", "Array<number>", "Array<object>", "Array<boolean>"], Field(default="Array<string>")]
+    input_mode: Annotated[Literal["constant", "variable"], Field(default="constant")]
+    value: Annotated[Any, Field(default=None)]
+    model_config = ConfigDict(extra="forbid")
+"""
 
 class IterationParam(ComponentParamBase):
     """

agent/component/llm.py

@@ -21,12 +21,12 @@ from copy import deepcopy
 from typing import Any, Generator
 import json_repair
 from functools import partial
-from api.db import LLMType
+from common.constants import LLMType
 from api.db.services.llm_service import LLMBundle
 from api.db.services.tenant_llm_service import TenantLLMService
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.api_utils import timeout
-from rag.prompts.generator import tool_call_summary, message_fit_in, citation_prompt
+from common.connection_utils import timeout
+from rag.prompts.generator import tool_call_summary, message_fit_in, citation_prompt, structured_output_prompt
 
 
 class LLMParam(ComponentParamBase):
@@ -214,10 +214,14 @@ class LLM(ComponentBase):
 
         prompt, msg, _ = self._prepare_prompt_variables()
         error: str = ""
-
-        if self._param.output_structure:
-            prompt += "\nThe output MUST follow this JSON format:\n"+json.dumps(self._param.output_structure, ensure_ascii=False, indent=2)
-            prompt += "\nRedundant information is FORBIDDEN."
+        output_structure=None
+        try:
+            output_structure = self._param.outputs['structured']
+        except Exception:
+            pass
+        if output_structure:
+            schema=json.dumps(output_structure, ensure_ascii=False, indent=2)
+            prompt += structured_output_prompt(schema)
             for _ in range(self._param.max_retries+1):
                 _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97))
                 error = ""
@@ -228,7 +232,7 @@ class LLM(ComponentBase):
                     error = ans
                     continue
                 try:
-                    self.set_output("structured_content", json_repair.loads(clean_formated_answer(ans)))
+                    self.set_output("structured", json_repair.loads(clean_formated_answer(ans)))
                     return
                 except Exception:
                     msg.append({"role": "user", "content": "The answer can't not be parsed as JSON"})
@@ -239,7 +243,7 @@ class LLM(ComponentBase):
 
         downstreams = self._canvas.get_component(self._id)["downstream"] if self._canvas.get_component(self._id) else []
         ex = self.exception_handler()
-        if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not self._param.output_structure and not (ex and ex["goto"]):
+        if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not output_structure and not (ex and ex["goto"]):
             self.set_output("content", partial(self._stream_output, prompt, msg))
             return
 

agent/component/message.py

@@ -23,7 +23,7 @@ from typing import Any
 from agent.component.base import ComponentBase, ComponentParamBase
 from jinja2 import Template as Jinja2Template
 
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class MessageParam(ComponentParamBase):
@@ -49,6 +49,9 @@ class MessageParam(ComponentParamBase):
 class Message(ComponentBase):
     component_name = "Message"
 
+    def get_input_elements(self) -> dict[str, Any]:
+        return self.get_input_elements_from_text("".join(self._param.content))
+
     def get_kwargs(self, script:str, kwargs:dict = {}, delimiter:str=None) -> tuple[str, dict[str, str | list | Any]]:
         for k,v in self.get_input_elements_from_text(script).items():
             if k in kwargs:

agent/component/string_transform.py

@@ -16,9 +16,11 @@
 import os
 import re
 from abc import ABC
+from typing import Any
+
 from jinja2 import Template as Jinja2Template
 from agent.component.base import ComponentParamBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 from .message import Message
 
 
@@ -43,6 +45,9 @@ class StringTransformParam(ComponentParamBase):
 class StringTransform(Message, ABC):
     component_name = "StringTransform"
 
+    def get_input_elements(self) -> dict[str, Any]:
+        return self.get_input_elements_from_text(self._param.script)
+
     def get_input_form(self) -> dict[str, dict]:
         if self._param.method == "split":
             return {

agent/component/switch.py

@@ -19,7 +19,7 @@ from abc import ABC
 from typing import Any
 
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class SwitchParam(ComponentParamBase):

agent/component/webhook.py

@@ -0,0 +1,38 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from agent.component.base import ComponentParamBase, ComponentBase
+
+
+class WebhookParam(ComponentParamBase):
+
+    """
+    Define the Begin component parameters.
+    """
+    def __init__(self):
+        super().__init__()
+
+    def get_input_form(self) -> dict[str, dict]:
+        return getattr(self, "inputs")
+
+
+class Webhook(ComponentBase):
+    component_name = "Webhook"
+
+    def _invoke(self, **kwargs):
+        pass
+
+    def thoughts(self) -> str:
+        return ""

agent/templates/advanced_ingestion_pipeline.json

@@ -2,10 +2,12 @@
     "id": 23,
     "title": {
         "en": "Advanced Ingestion Pipeline",
+        "de": "Erweiterte Ingestion Pipeline",
         "zh": "编排复杂的 Ingestion Pipeline"
     },
     "description": {
         "en": "This template demonstrates how to use an LLM to generate summaries, keywords, Q&A, and metadata for each chunk to support diverse retrieval needs.",
+        "de": "Diese Vorlage demonstriert, wie ein LLM verwendet wird, um Zusammenfassungen, Schlüsselwörter, Fragen & Antworten und Metadaten für jedes Segment zu generieren, um vielfältige Abrufanforderungen zu unterstützen.",
         "zh": "此模板演示如何利用大模型为切片生成摘要、关键词、问答及元数据，以满足多样化的召回需求。"
     },
     "canvas_type": "Ingestion Pipeline",

agent/templates/chunk_summary.json

@@ -2,10 +2,12 @@
     "id": 24,
     "title": {
         "en": "Chunk Summary",
+        "de": "Segmentzusammenfassung",
         "zh": "总结切片"
     },
     "description": {
         "en": "This template uses an LLM to generate chunk summaries for building text and vector indexes. During retrieval, summaries enhance matching, and the original chunks are returned as results.",
+        "de": "Diese Vorlage nutzt ein LLM zur Generierung von Segmentzusammenfassungen für den Aufbau von Text- und Vektorindizes. Bei der Abfrage verbessern die Zusammenfassungen die Übereinstimmung, während die ursprünglichen Segmente als Ergebnisse zurückgegeben werden.",
         "zh": "此模板利用大模型生成切片摘要，并据此建立全文索引与向量。检索时以摘要提升匹配效果，最终召回对应的原文切片。"
     },
     "canvas_type": "Ingestion Pipeline",

agent/templates/generate_SEO_blog.json

@@ -2,9 +2,11 @@
     "id": 8,
     "title": {
         "en": "Generate SEO Blog",
+        "de": "SEO Blog generieren",
         "zh": "生成SEO博客"},
     "description": {
         "en": "This is a multi-agent version of the SEO blog generation workflow. It simulates a small team of AI “writers”, where each agent plays a specialized role — just like a real editorial team.",
+        "de": "Dies ist eine Multi-Agenten-Version des Workflows zur Erstellung von SEO-Blogs. Sie simuliert ein kleines Team von KI-„Autoren“, in dem jeder Agent eine spezielle Rolle übernimmt – genau wie in einem echten Redaktionsteam.",
         "zh": "多智能体架构可根据简单的用户输入自动生成完整的SEO博客文章。模拟小型“作家”团队，其中每个智能体扮演一个专业角色——就像真正的编辑团队。"},
     "canvas_type": "Agent",
     "dsl": {

agent/templates/knowledge_base_report.json

@@ -2,9 +2,11 @@
     "id": 20,
     "title": {
         "en": "Report Agent Using Knowledge Base",
+        "de": "Berichtsagent mit Wissensdatenbank",
         "zh": "知识库检索智能体"},
     "description": {
         "en": "A report generation assistant using local knowledge base, with advanced capabilities in task planning, reasoning, and reflective analysis. Recommended for academic research paper Q&A",
+        "de": "Ein Berichtsgenerierungsassistent, der eine lokale Wissensdatenbank nutzt, mit erweiterten Fähigkeiten in Aufgabenplanung, Schlussfolgerung und reflektierender Analyse. Empfohlen für akademische Forschungspapier-Fragen und -Antworten.",
         "zh": "一个使用本地知识库的报告生成助手，具备高级能力，包括任务规划、推理和反思性分析。推荐用于学术研究论文问答。"},
     "canvas_type": "Agent",
     "dsl": {

agent/templates/knowledge_base_report_r.json

@@ -1,10 +1,12 @@
 {
     "id": 21,
     "title": {
-        "en": "Report Agent Using Knowledge Base", 
+        "en": "Report Agent Using Knowledge Base",
+        "de": "Berichtsagent mit Wissensdatenbank",
         "zh": "知识库检索智能体"},
     "description": {
         "en": "A report generation assistant using local knowledge base, with advanced capabilities in task planning, reasoning, and reflective analysis. Recommended for academic research paper Q&A",
+        "de": "Ein Berichtsgenerierungsassistent, der eine lokale Wissensdatenbank nutzt, mit erweiterten Fähigkeiten in Aufgabenplanung, Schlussfolgerung und reflektierender Analyse. Empfohlen für akademische Forschungspapier-Fragen und -Antworten.",
         "zh": "一个使用本地知识库的报告生成助手，具备高级能力，包括任务规划、推理和反思性分析。推荐用于学术研究论文问答。"},
     "canvas_type": "Recommended",
     "dsl": {

agent/templates/market_generate_seo_blog.json

@@ -2,9 +2,11 @@
     "id": 12,
     "title": {
         "en": "Generate SEO Blog",
+        "de": "SEO Blog generieren",
         "zh": "生成SEO博客"},
     "description": {
-        "en": "This workflow automatically generates a complete SEO-optimized blog article based on a simple user input. You don’t need any writing experience. Just provide a topic or short request — the system will handle the rest.",
+        "en": "This workflow automatically generates a complete SEO-optimized blog article based on a simple user input. You don't need any writing experience. Just provide a topic or short request — the system will handle the rest.",
+        "de": "Dieser Workflow generiert automatisch einen vollständigen SEO-optimierten Blogartikel basierend auf einer einfachen Benutzereingabe. Sie benötigen keine Schreiberfahrung. Geben Sie einfach ein Thema oder eine kurze Anfrage ein – das System übernimmt den Rest.",
         "zh": "此工作流根据简单的用户输入自动生成完整的SEO博客文章。你无需任何写作经验，只需提供一个主题或简短请求，系统将处理其余部分。"},
     "canvas_type": "Marketing",
     "dsl": {
@@ -916,4 +918,4 @@
             "retrieval": []
         },
     "avatar": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAwADADASIAAhEBAxEB/8QAGQAAAwEBAQAAAAAAAAAAAAAABgkKBwUI/8QAMBAAAAYCAQIEBQQCAwAAAAAAAQIDBAUGBxEhCAkAEjFBFFFhcaETFiKRFyOx8PH/xAAaAQACAwEBAAAAAAAAAAAAAAACAwABBgQF/8QALBEAAgIBAgUCBAcAAAAAAAAAAQIDBBEFEgATITFRIkEGIzJhFBUWgaGx8P/aAAwDAQACEQMRAD8AfF2hez9089t7pvxgQMa1Gb6qZ6oQE9m/NEvCIStyPfJSOF/M1epzMugo/qtMqbiRc1mJjoJKCLMNIxKcsLJedfO1Ct9cI63x9fx6CA/19t+oh4LFA5HfuAgP/A8eOIsnsTBrkBHXA7+v53+Q+ficTgJft9gIgA+/P9/1r342O/YA8A8k3/if+IbAN7+2/f8AAiI6H19PGoPyESTMZQPKUAHkQEN+3r9dh78/YPGUTk2wb/qAZZIugH1OHH5DjkdfbnWw2DsOxPj+xjrnx2H39unBopJGBn9s+PHv1HXjPJtH+J+B40O9a16h/wB/92j/ALrPa/wR104UyAobHlXhuo2HrEtK4qy3CwjKOuJLRHJLSkXWrFKs/gVrJVrE8TUiH8bPrP20UEu8m4hNpMJJuTOfnbUw/kUqyZgMHGjAO9+mtDsQ53sdcB6eMhnpEjhNQxRKICAgHy5+/roOdjr7c+J6O4x07dx484/n7nzw1gexBGfIPkZ/3t39uGpqc6+fP5/Ht8vGFZCzJjWpWuBxvO2yPjrtclUUK7BqmUI4fuASeyhG5FzFI0Bw4aQ0iZNoDgzvRW4qtyFkI4XmwyEk2YNnDp0sVBu3IUyy5iqH8gqKERSIRNIii67hddRJs1at01Xbx2sgzZoLu10UFJR+4V1A5cxF3FqNcLvjwcno43uuLrOxZYjujaClcb4QQfxEizpFiQyM9olcueRnjC2ZMt9iY06zL0qytrMSqSOVGsfHMaGhZ3l4lSRI2MqE74zJvRTveNFWWIh3RWw+XCAM5icKQLrCH57T17FhErSlRXnWvyZXKQwWJ3eraD14p5YuZCFgacskK2oGkVuKO5GYTHzf7DaD12cBD3DgPOIDrWw9PnrXPgDkpVsUDGMG+DD6E9gHXIjrYjwUPQTCXYgHPhIV974+F6E1hpC14Yzmzj56YaQEeZhXsayD1zLPW7pygxaMf81Nzu1iJsnIuDIKnaJAkPldqrHaoORZ73tMVEbFdSXT9nVgRQgnBq6j8e/HCIEATpAnH5KlmRVkFRFJwks/bqImSXJ5VFyA3N6Ikh3bCW3YHp5cowOmCfTgA+xJCnrjtwHKcLvJj2ZGcTRFj19kEhckdzgEjKnABGSSzdc1Fe5byXXGNjKdvRcw5NxvLidNZFFCxUa62KrzMaChw8hhYScFJtROAgmuLByq1MsgkZYPaVVuDe0wraRaqAdJwgRQo+YR8xTlAQNx6b49w41vXiJpCalLh1jZhyrTqRM4+jstdRmYryNkydLQRWg1LNGcWd5jIFFvCythlIySa0mNu74sKRQtaWsTmupqPItw0lE52ufpyYzrSkx6cw5bLmBEpkTsz+dt8P5QFuCRtAIkBH9MuwKHICIaDQhnojMs9mKaeGcrMxXlQtAYkdVljimRrE5MqI4zL8oSqQ6wxjodBqK05qdK3Vo3aCSVkBW7bjuC1NFJJBPaqyx6fp6pWkliYLXK2XrukkRu2CCVoSWMgsdMyySKwoLFcIGWSTUMg4IBgTcICoBhRcplMcpFkhIqQp1ClMBTmA0Zfe1zpjvHfXff65bZlzXpB3jjGTgiirmPjAfs16PHqHeQ75Wbj3xxZpOEkV3LRJJSPdomUBZISJLncV2k+8D07dxXp7xsYuTapA9UkJUYWIzNhadnWEZeCXGLQQiJi1ViHfhHL2unWh+mlORsrW0JFpEFnGVfm1mU4kq0FY3eD6corJncv6dr5NLSMNXVaTUksjTiMnaq8uFfSVuDyiJ1iZpy0LOJtpa3YfkcQ5fdozyxI2m5qqcrHN61YYmHsh6v3o9ParYmYJEtlhIx6+gUbjgD23M6oqg92YL0JyF6Bps+qDValVA9h9Lj5SZI3SHXdEQlj1wiQtLLIe6pGzjO3BlBkK1hxpblLVH5wdW0BcFKf/JwRtjsot2z8omaSdxbzzk1iEjsE0AM9rrRZNRIrVyo7dGO6E+oh8axLlJ5H5VaJKx7ePRGFbW6vUeFfHQIWPTI9Tm7HHfuhqY7E6C7JFqUzM6iZXIoncNxX7+bIVdJnTT48x3OQU1krIDW3UeixVhyISzYz6cadY5Xph6TseRNTRsTElzzBn9Vlly0TAERsdgnMYyLROjyFbg5R4ZlsGaMT4yNi2Zlq1GwjZB3jq0PsaJfA3t0jL0W0Y9xf1V41lpWckXMLaZiwxuKYPqc6LlHdkeRF+Qxswx5ASDqBVrsL+2A/N6SiCbYymV2BywJiMZj3GRRMTnL+lVyHCll3R7Szv0vqXMtQ74T+HijljIScLaEpkKCB3rqMBIi0jPs5JeOKTZMZEi5VVnouzy0k3jXjWSMlY6UcVGDxlKMVDqx91SILWSi3D2KdgYy3kP8E9X/AE1SnRXBNdNRMlefT6g7aY6giK+cPLGNg0bY68rcnpsNh9PqIBve/EcPQ3WIq2dR93xpSgk5SAZ9R6MLAOZFUkpLSUDXp6/KPpGUkmTdswlnKnwbl5ITMdGwcXJi7LKsqzUmT5tWYmkXuF9wjBvb76b7dHheazJ9RElUJOCxViuMlUJC0Gtz6PKyjLBY4qMWUe12r1xZ6lOyT6XPEBKN2CkTDOlZd02TBdTMt7Upx2knrkdCv1UKjDKn1A7XBYH6SCOOrWn5Oi/DtRiu+GleRthDL8rXdVjZlcfWrSIxVlGGGCOnH//Z"
-}
+}

agent/templates/seo_blog.json

@@ -2,9 +2,11 @@
     "id": 4,
     "title": {
         "en": "Generate SEO Blog",
+        "de": "SEO Blog generieren",
         "zh": "生成SEO博客"},
     "description": {
-        "en": "This workflow automatically generates a complete SEO-optimized blog article based on a simple user input. You don’t need any writing experience. Just provide a topic or short request — the system will handle the rest.",
+        "en": "This workflow automatically generates a complete SEO-optimized blog article based on a simple user input. You don't need any writing experience. Just provide a topic or short request — the system will handle the rest.",
+        "de": "Dieser Workflow generiert automatisch einen vollständigen SEO-optimierten Blogartikel basierend auf einer einfachen Benutzereingabe. Sie benötigen keine Schreiberfahrung. Geben Sie einfach ein Thema oder eine kurze Anfrage ein – das System übernimmt den Rest.",
         "zh": "此工作流根据简单的用户输入自动生成完整的SEO博客文章。你无需任何写作经验，只需提供一个主题或简短请求，系统将处理其余部分。"},
     "canvas_type": "Recommended",
     "dsl": {
@@ -916,4 +918,4 @@
             "retrieval": []
         },
     "avatar": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAwADADASIAAhEBAxEB/8QAGQAAAwEBAQAAAAAAAAAAAAAABgkKBwUI/8QAMBAAAAYCAQIEBQQCAwAAAAAAAQIDBAUGBxEhCAkAEjFBFFFhcaETFiKRFyOx8PH/xAAaAQACAwEBAAAAAAAAAAAAAAACAwABBgQF/8QALBEAAgIBAgUCBAcAAAAAAAAAAQIDBBEFEgATITFRIkEGIzJhFBUWgaGx8P/aAAwDAQACEQMRAD8AfF2hez9089t7pvxgQMa1Gb6qZ6oQE9m/NEvCIStyPfJSOF/M1epzMugo/qtMqbiRc1mJjoJKCLMNIxKcsLJedfO1Ct9cI63x9fx6CA/19t+oh4LFA5HfuAgP/A8eOIsnsTBrkBHXA7+v53+Q+ficTgJft9gIgA+/P9/1r342O/YA8A8k3/if+IbAN7+2/f8AAiI6H19PGoPyESTMZQPKUAHkQEN+3r9dh78/YPGUTk2wb/qAZZIugH1OHH5DjkdfbnWw2DsOxPj+xjrnx2H39unBopJGBn9s+PHv1HXjPJtH+J+B40O9a16h/wB/92j/ALrPa/wR104UyAobHlXhuo2HrEtK4qy3CwjKOuJLRHJLSkXWrFKs/gVrJVrE8TUiH8bPrP20UEu8m4hNpMJJuTOfnbUw/kUqyZgMHGjAO9+mtDsQ53sdcB6eMhnpEjhNQxRKICAgHy5+/roOdjr7c+J6O4x07dx484/n7nzw1gexBGfIPkZ/3t39uGpqc6+fP5/Ht8vGFZCzJjWpWuBxvO2yPjrtclUUK7BqmUI4fuASeyhG5FzFI0Bw4aQ0iZNoDgzvRW4qtyFkI4XmwyEk2YNnDp0sVBu3IUyy5iqH8gqKERSIRNIii67hddRJs1at01Xbx2sgzZoLu10UFJR+4V1A5cxF3FqNcLvjwcno43uuLrOxZYjujaClcb4QQfxEizpFiQyM9olcueRnjC2ZMt9iY06zL0qytrMSqSOVGsfHMaGhZ3l4lSRI2MqE74zJvRTveNFWWIh3RWw+XCAM5icKQLrCH57T17FhErSlRXnWvyZXKQwWJ3eraD14p5YuZCFgacskK2oGkVuKO5GYTHzf7DaD12cBD3DgPOIDrWw9PnrXPgDkpVsUDGMG+DD6E9gHXIjrYjwUPQTCXYgHPhIV974+F6E1hpC14Yzmzj56YaQEeZhXsayD1zLPW7pygxaMf81Nzu1iJsnIuDIKnaJAkPldqrHaoORZ73tMVEbFdSXT9nVgRQgnBq6j8e/HCIEATpAnH5KlmRVkFRFJwks/bqImSXJ5VFyA3N6Ikh3bCW3YHp5cowOmCfTgA+xJCnrjtwHKcLvJj2ZGcTRFj19kEhckdzgEjKnABGSSzdc1Fe5byXXGNjKdvRcw5NxvLidNZFFCxUa62KrzMaChw8hhYScFJtROAgmuLByq1MsgkZYPaVVuDe0wraRaqAdJwgRQo+YR8xTlAQNx6b49w41vXiJpCalLh1jZhyrTqRM4+jstdRmYryNkydLQRWg1LNGcWd5jIFFvCythlIySa0mNu74sKRQtaWsTmupqPItw0lE52ufpyYzrSkx6cw5bLmBEpkTsz+dt8P5QFuCRtAIkBH9MuwKHICIaDQhnojMs9mKaeGcrMxXlQtAYkdVljimRrE5MqI4zL8oSqQ6wxjodBqK05qdK3Vo3aCSVkBW7bjuC1NFJJBPaqyx6fp6pWkliYLXK2XrukkRu2CCVoSWMgsdMyySKwoLFcIGWSTUMg4IBgTcICoBhRcplMcpFkhIqQp1ClMBTmA0Zfe1zpjvHfXff65bZlzXpB3jjGTgiirmPjAfs16PHqHeQ75Wbj3xxZpOEkV3LRJJSPdomUBZISJLncV2k+8D07dxXp7xsYuTapA9UkJUYWIzNhadnWEZeCXGLQQiJi1ViHfhHL2unWh+mlORsrW0JFpEFnGVfm1mU4kq0FY3eD6corJncv6dr5NLSMNXVaTUksjTiMnaq8uFfSVuDyiJ1iZpy0LOJtpa3YfkcQ5fdozyxI2m5qqcrHN61YYmHsh6v3o9ParYmYJEtlhIx6+gUbjgD23M6oqg92YL0JyF6Bps+qDValVA9h9Lj5SZI3SHXdEQlj1wiQtLLIe6pGzjO3BlBkK1hxpblLVH5wdW0BcFKf/JwRtjsot2z8omaSdxbzzk1iEjsE0AM9rrRZNRIrVyo7dGO6E+oh8axLlJ5H5VaJKx7ePRGFbW6vUeFfHQIWPTI9Tm7HHfuhqY7E6C7JFqUzM6iZXIoncNxX7+bIVdJnTT48x3OQU1krIDW3UeixVhyISzYz6cadY5Xph6TseRNTRsTElzzBn9Vlly0TAERsdgnMYyLROjyFbg5R4ZlsGaMT4yNi2Zlq1GwjZB3jq0PsaJfA3t0jL0W0Y9xf1V41lpWckXMLaZiwxuKYPqc6LlHdkeRF+Qxswx5ASDqBVrsL+2A/N6SiCbYymV2BywJiMZj3GRRMTnL+lVyHCll3R7Szv0vqXMtQ74T+HijljIScLaEpkKCB3rqMBIi0jPs5JeOKTZMZEi5VVnouzy0k3jXjWSMlY6UcVGDxlKMVDqx91SILWSi3D2KdgYy3kP8E9X/AE1SnRXBNdNRMlefT6g7aY6giK+cPLGNg0bY68rcnpsNh9PqIBve/EcPQ3WIq2dR93xpSgk5SAZ9R6MLAOZFUkpLSUDXp6/KPpGUkmTdswlnKnwbl5ITMdGwcXJi7LKsqzUmT5tWYmkXuF9wjBvb76b7dHheazJ9RElUJOCxViuMlUJC0Gtz6PKyjLBY4qMWUe12r1xZ6lOyT6XPEBKN2CkTDOlZd02TBdTMt7Upx2knrkdCv1UKjDKn1A7XBYH6SCOOrWn5Oi/DtRiu+GleRthDL8rXdVjZlcfWrSIxVlGGGCOnH//Z"
-}
+}

agent/templates/sql_assistant.json

@@ -2,10 +2,12 @@
     "id": 17,
     "title": {
         "en": "SQL Assistant",
+        "de": "SQL Assistent",
         "zh": "SQL助理"},
     "description": {
-        "en": "SQL Assistant is an AI-powered tool that lets business users turn plain-English questions into fully formed SQL queries. Simply type your question (e.g., “Show me last quarter’s top 10 products by revenue”) and SQL Assistant generates the exact SQL, runs it against your database, and returns the results in seconds. ",
-        "zh": "用户能够将简单文本问题转化为完整的SQL查询并输出结果。只需输入您的问题（例如，“展示上个季度前十名按收入排序的产品”），SQL助理就会生成精确的SQL语句，对其运行您的数据库，并几秒钟内返回结果。"},
+        "en": "SQL Assistant is an AI-powered tool that lets business users turn plain-English questions into fully formed SQL queries. Simply type your question (e.g., 'Show me last quarter's top 10 products by revenue') and SQL Assistant generates the exact SQL, runs it against your database, and returns the results in seconds. ",
+        "de": "SQL-Assistent ist ein KI-gestütztes Tool, mit dem Geschäftsanwender einfache englische Fragen in vollständige SQL-Abfragen umwandeln können. Geben Sie einfach Ihre Frage ein (z.B. 'Zeige mir die Top 10 Produkte des letzten Quartals nach Umsatz') und der SQL-Assistent generiert das exakte SQL, führt es gegen Ihre Datenbank aus und liefert die Ergebnisse in Sekunden.",
+        "zh": "用户能够将简单文本问题转化为完整的SQL查询并输出结果。只需输入您的问题（例如，展示上个季度前十名按收入排序的产品），SQL助理就会生成精确的SQL语句，对其运行您的数据库，并几秒钟内返回结果。"},
     "canvas_type": "Marketing",
     "dsl": {
             "components": {
@@ -713,4 +715,4 @@
             "retrieval": []
         },
     "avatar": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAcFBQYFBAcGBQYIBwcIChELCgkJChUPEAwRGBUaGRgVGBcbHichGx0lHRcYIi4iJSgpKywrGiAvMy8qMicqKyr/2wBDAQcICAoJChQLCxQqHBgcKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKir/wAARCAAwADADAREAAhEBAxEB/8QAGgAAAwEBAQEAAAAAAAAAAAAABQYHBAMAAf/EADIQAAEDAwMCBAMHBQAAAAAAAAECAwQFESEABjESEyJBUYEUYXEHFSNSkaGxMjNictH/xAAZAQADAQEBAAAAAAAAAAAAAAACAwQBAAX/xAAlEQACAgICAgEEAwAAAAAAAAABAgARAyESMQRBEyIycYFCkbH/2gAMAwEAAhEDEQA/AKHt2DGpNHXDLrZdWtSrIub39tZ5GbGwPA+pmDFkX7x7idvra85xqQaFNkxUTVIVJQzf8QpBFjbgEenNs681MnA9WJ6fEOKJoxVpSpFLTCo6KEZlTlLcQBIJS20hAv1D1ve+qPk52b0IsYuIGtyt7ZkVVNP+H3A5GdlN2u7GQUBSfmkk8cXH10tmLD6Yl0CG5qmTXBMZiQEMuvupUoKdc6UeEi4FsqOeBxrsKnv1AY+hJ2l5yfu6qQ6/UZtPDRHZ+Eldpsqz1hSrXJGLXwRxqxUQizFs7galPYUFDKT+h15oMuImspQpFiL+2i1A3A1bgxmixUgwlT8ZfgJ/y8P8HXdRuPZoxaqtfkQKbKqF03jtEoDeFKV1lNgfK4H764XfccVUgipvdiwKpFaXMLklFg4juuqV0m3Izg/MaEZCDYMScYqiJOd6xmqfUVfBJcWwtHV1Elfi87k51ViyhrsxL4ivQj1KrFZjTGjTJ8aShdyph5SUqFhwPzX9jpC0dXUqZK3ViHNq7oNaVJjz2Vw5LCrdKknpULZyfMf801MfI1e5NmpAGHUL12EZNFWWlhXSUuWHKgk3xomwEDuDhzLysySU9EndEVyIz3GmxJR+KpBIdCLlRHn/AFEjjIF9AMJlZ8gLZ/qUiJSg1Tu0HO4plFj4FC1h9NYfHIU7kwzgnqCJlKLiCO2s6hKytWiPJoFdfnLW7HS0or6bqXbjg2AI99XjAa3NPlL6jFTduOR5sd1+oyfjQMONqI7QOMA4V7/pqjHjC9SLNn56I1HiqrqTUKM0hbq2lpst5CQSST54xjSPJbICOHUhawISiRQ02T2Uq6AAkqFj/GquJQks1iEr/INLU82bploKSFXusG9xfjHofXQuQUNRoQqQT0ZwVEST5687iZWGgpDsebNbaTDfKVL/ALnbQU/UkKNhjXpFt0BJBVXe/wAGGG6YMlvvNkjlBGmKeJimHIVc0TY89akCKspT28C5BKgDyR7fvrCFI+q/1DQsvVfudYcVyKw49KU6tZyQbmwHFhrOKr9s0uz0CAIpbr3RKo1Rbh02C4HJISp2ZIz0pJ8IQk5Nr/QXznSX6NSnGAwHI/gD/TM+3vtAj1arJpcpgtPdPSH0kFt5wDxAWOOLgamIAFwijCfD927N2tGXuNxlK2W0occUhJWpR+QzzrPjc+pvyqT3Ftf2zbObf7YYecb6CrrDAGfy20wYMkA5Vjbtev7b3nEcXRela27d1ogoWi/rnQsjrqZzHdwzKoKUsqWz3mOnJUlZJt8uokD621w+RdzgynUkUpoUafPZXMnSHlrKluyX1Eug8XF7GwxbgWxrubMO5WmNRsCKtLfcY3rAU0nIltkBP+w0X8Jjdz//2Q=="
-}
+}

agent/templates/title_chunker.json

@@ -2,10 +2,12 @@
     "id": 25,
     "title": {
         "en": "Title Chunker",
+        "de": "Titel basierte Segmentierung",
         "zh": "标题切片"
     },
     "description": {
         "en": "This template slices the parsed file based on its title structure. It is ideal for documents with well-defined headings, such as product manuals, legal contracts, research reports, and academic papers.",
+        "de": "Diese Vorlage segmentiert die geparste Datei basierend auf ihrer Titelstruktur. Sie eignet sich ideal für Dokumente mit klar definierten Überschriften, wie Produkthandbücher, Verträge, Forschungsberichte und wissenschaftliche Arbeiten.",
         "zh": "此模板将解析后的文件按标题结构进行切片，适用于具有清晰标题层级的文档类型，如产品手册、合同法规、研究报告和学术论文等。"
     },
     "canvas_type": "Ingestion Pipeline",

agent/test/client.py

@@ -16,7 +16,7 @@
 import argparse
 import os
 from agent.canvas import Canvas
-from api import settings
+from common import settings
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

agent/tools/arxiv.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 import arxiv
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class ArXivParam(ToolParamBase):

agent/tools/base.py

@@ -20,7 +20,7 @@ from copy import deepcopy
 from functools import partial
 from typing import TypedDict, List, Any
 from agent.component.base import ComponentParamBase, ComponentBase
-from api.utils import hash_str2int
+from common.misc_utils import hash_str2int
 from rag.llm.chat_model import ToolCallSession
 from rag.prompts.generator import kb_prompt
 from rag.utils.mcp_tool_call_conn import MCPToolCallSession

agent/tools/code_exec.py

@@ -21,8 +21,8 @@ from strenum import StrEnum
 from typing import Optional
 from pydantic import BaseModel, Field, field_validator
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api import settings
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
+from common import settings
 
 
 class Language(StrEnum):

agent/tools/duckduckgo.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 from duckduckgo_search import DDGS
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class DuckDuckGoParam(ToolParamBase):

agent/tools/email.py

@@ -25,7 +25,7 @@ from email.header import Header
 from email.utils import formataddr
 
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class EmailParam(ToolParamBase):

agent/tools/exesql.py

@@ -22,7 +22,7 @@ import pymysql
 import psycopg2
 import pyodbc
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class ExeSQLParam(ToolParamBase):

agent/tools/github.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 import requests
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class GitHubParam(ToolParamBase):

agent/tools/google.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 from serpapi import GoogleSearch
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class GoogleParam(ToolParamBase):

agent/tools/googlescholar.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 from scholarly import scholarly
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class GoogleScholarParam(ToolParamBase):

agent/tools/pubmed.py

@@ -21,7 +21,7 @@ from Bio import Entrez
 import re
 import xml.etree.ElementTree as ET
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class PubMedParam(ToolParamBase):

agent/tools/retrieval.py

@@ -13,17 +13,19 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+from functools import partial
+import json
 import os
 import re
 from abc import ABC
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api.db import LLMType
+from common.constants import LLMType
 from api.db.services.document_service import DocumentService
 from api.db.services.dialog_service import meta_filter
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
-from api import settings
-from api.utils.api_utils import timeout
+from common import settings
+from common.connection_utils import timeout
 from rag.app.tag import label_question
 from rag.prompts.generator import cross_languages, kb_prompt, gen_meta_filter
 
@@ -131,7 +133,35 @@ class Retrieval(ToolBase, ABC):
                 if not doc_ids:
                     doc_ids = None
             elif self._param.meta_data_filter.get("method") == "manual":
-                doc_ids.extend(meta_filter(metas, self._param.meta_data_filter["manual"]))
+                filters=self._param.meta_data_filter["manual"]
+                for flt in filters:
+                    pat = re.compile(r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z:0-9_.-]+|sys\.[a-z_]+)\} *\}*")
+                    s = flt["value"]
+                    out_parts = []
+                    last = 0
+
+                    for m in pat.finditer(s):
+                        out_parts.append(s[last:m.start()])
+                        key = m.group(1)
+                        v = self._canvas.get_variable_value(key)
+                        if v is None:
+                            rep = ""
+                        elif isinstance(v, partial):
+                            buf = []
+                            for chunk in v():
+                                buf.append(chunk)
+                            rep = "".join(buf)
+                        elif isinstance(v, str):
+                            rep = v
+                        else:
+                            rep = json.dumps(v, ensure_ascii=False)
+
+                        out_parts.append(rep)
+                        last = m.end()
+
+                    out_parts.append(s[last:])
+                    flt["value"] = "".join(out_parts)
+                doc_ids.extend(meta_filter(metas, filters))
                 if not doc_ids:
                     doc_ids = None
 

agent/tools/searxng.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 import requests
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class SearXNGParam(ToolParamBase):

agent/tools/tavily.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 from tavily import TavilyClient
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class TavilySearchParam(ToolParamBase):

agent/tools/wencai.py

@@ -21,7 +21,7 @@ import pandas as pd
 import pywencai
 
 from agent.tools.base import ToolParamBase, ToolMeta, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class WenCaiParam(ToolParamBase):

agent/tools/wikipedia.py

@@ -19,7 +19,7 @@ import time
 from abc import ABC
 import wikipedia
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class WikipediaParam(ToolParamBase):

agent/tools/yahoofinance.py

@@ -20,7 +20,7 @@ from abc import ABC
 import pandas as pd
 import yfinance as yf
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
-from api.utils.api_utils import timeout
+from common.connection_utils import timeout
 
 
 class YahooFinanceParam(ToolParamBase):

api/apps/__init__.py

@@ -24,16 +24,16 @@ from flask_cors import CORS
 from flasgger import Swagger
 from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
 
-from api.db import StatusEnum
+from common.constants import StatusEnum
 from api.db.db_models import close_connection
 from api.db.services import UserService
-from api.utils.json import CustomJSONEncoder
+from api.utils.json_encode import CustomJSONEncoder
 from api.utils import commands
 
 from flask_mail import Mail
 from flask_session import Session
 from flask_login import LoginManager
-from api import settings
+from common import settings
 from api.utils.api_utils import server_error_response
 from api.constants import API_VERSION
 

api/apps/api_app.py

@@ -21,7 +21,7 @@ from flask import request, Response
 from api.db.services.llm_service import LLMBundle
 from flask_login import login_required, current_user
 
-from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileType, LLMType, ParserType, FileSource
+from api.db import VALID_FILE_TYPES, FileType
 from api.db.db_models import APIToken, Task, File
 from api.db.services import duplicate_name
 from api.db.services.api_service import APITokenService, API4ConversationService
@@ -32,20 +32,21 @@ from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import queue_tasks, TaskService
 from api.db.services.user_service import UserTenantService
-from api import settings
-from api.utils import get_uuid, current_timestamp, datetime_format
+from common.misc_utils import get_uuid
+from common.constants import RetCode, VALID_TASK_STATUS, LLMType, ParserType, FileSource
 from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request, \
     generate_confirmation_token
 
 from api.utils.file_utils import filename_type, thumbnail
 from rag.app.tag import label_question
 from rag.prompts.generator import keyword_extraction
-from rag.utils.storage_factory import STORAGE_IMPL
+from common.time_utils import current_timestamp, datetime_format
 
 from api.db.services.canvas_service import UserCanvasService
 from agent.canvas import Canvas
 from functools import partial
 from pathlib import Path
+from common import settings
 
 
 @manager.route('/new_token', methods=['POST'])  # noqa: F821
@@ -58,7 +59,7 @@ def new_token():
             return get_data_error_result(message="Tenant not found!")
 
         tenant_id = tenants[0].tenant_id
-        obj = {"tenant_id": tenant_id, "token": generate_confirmation_token(tenant_id),
+        obj = {"tenant_id": tenant_id, "token": generate_confirmation_token(),
                "create_time": current_timestamp(),
                "create_date": datetime_format(datetime.now()),
                "update_time": None,
@@ -144,7 +145,7 @@ def set_conversation():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
     try:
         if objs[0].source == "agent":
             e, cvs = UserCanvasService.get_by_id(objs[0].dialog_id)
@@ -185,7 +186,7 @@ def completion():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
     req = request.json
     e, conv = API4ConversationService.get_by_id(req["conversation_id"])
     if not e:
@@ -351,7 +352,7 @@ def get_conversation(conversation_id):
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     try:
         e, conv = API4ConversationService.get_by_id(conversation_id)
@@ -361,7 +362,7 @@ def get_conversation(conversation_id):
         conv = conv.to_dict()
         if token != APIToken.query(dialog_id=conv['dialog_id'])[0].token:
             return get_json_result(data=False, message='Authentication error: API key is invalid for this conversation_id!"',
-                                   code=settings.RetCode.AUTHENTICATION_ERROR)
+                                   code=RetCode.AUTHENTICATION_ERROR)
 
         for referenct_i in conv['reference']:
             if referenct_i is None or len(referenct_i) == 0:
@@ -382,7 +383,7 @@ def upload():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     kb_name = request.form.get("kb_name").strip()
     tenant_id = objs[0].tenant_id
@@ -398,12 +399,12 @@ def upload():
 
     if 'file' not in request.files:
         return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+            data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
 
     file = request.files['file']
     if file.filename == '':
         return get_json_result(
-            data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
+            data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR)
 
     root_folder = FileService.get_root_folder(tenant_id)
     pf_id = root_folder["id"]
@@ -426,10 +427,10 @@ def upload():
                 message="This type of file has not been supported yet!")
 
         location = filename
-        while STORAGE_IMPL.obj_exist(kb_id, location):
+        while settings.STORAGE_IMPL.obj_exist(kb_id, location):
             location += "_"
         blob = request.files['file'].read()
-        STORAGE_IMPL.put(kb_id, location, blob)
+        settings.STORAGE_IMPL.put(kb_id, location, blob)
         doc = {
             "id": get_uuid(),
             "kb_id": kb.id,
@@ -495,17 +496,17 @@ def upload_parse():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     if 'file' not in request.files:
         return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+            data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
 
     file_objs = request.files.getlist('file')
     for file_obj in file_objs:
         if file_obj.filename == '':
             return get_json_result(
-                data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
+                data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR)
 
     doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, objs[0].tenant_id)
     return get_json_result(data=doc_ids)
@@ -518,7 +519,7 @@ def list_chunks():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
 
@@ -558,7 +559,7 @@ def get_chunk(chunk_id):
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
     try:
         tenant_id = objs[0].tenant_id
         kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
@@ -583,7 +584,7 @@ def list_kb_docs():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
     tenant_id = objs[0].tenant_id
@@ -636,7 +637,7 @@ def docinfos():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
     req = request.json
     doc_ids = req["doc_ids"]
     docs = DocumentService.get_by_ids(doc_ids)
@@ -650,7 +651,7 @@ def document_rm():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     tenant_id = objs[0].tenant_id
     req = request.json
@@ -697,12 +698,12 @@ def document_rm():
             FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
             File2DocumentService.delete_by_document_id(doc_id)
 
-            STORAGE_IMPL.rm(b, n)
+            settings.STORAGE_IMPL.rm(b, n)
         except Exception as e:
             errors += str(e)
 
     if errors:
-        return get_json_result(data=False, message=errors, code=settings.RetCode.SERVER_ERROR)
+        return get_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
 
     return get_json_result(data=True)
 
@@ -717,7 +718,7 @@ def completion_faq():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     e, conv = API4ConversationService.get_by_id(req["conversation_id"])
     if not e:
@@ -790,7 +791,7 @@ def completion_faq():
                 if ans["reference"]["chunks"][chunk_idx]["img_id"]:
                     try:
                         bkt, nm = ans["reference"]["chunks"][chunk_idx]["img_id"].split("-")
-                        response = STORAGE_IMPL.get(bkt, nm)
+                        response = settings.STORAGE_IMPL.get(bkt, nm)
                         data_type_picture["url"] = base64.b64encode(response).decode('utf-8')
                         data.append(data_type_picture)
                         break
@@ -835,7 +836,7 @@ def completion_faq():
             if ans["reference"]["chunks"][chunk_idx]["img_id"]:
                 try:
                     bkt, nm = ans["reference"]["chunks"][chunk_idx]["img_id"].split("-")
-                    response = STORAGE_IMPL.get(bkt, nm)
+                    response = settings.STORAGE_IMPL.get(bkt, nm)
                     data_type_picture["url"] = base64.b64encode(response).decode('utf-8')
                     data.append(data_type_picture)
                     break
@@ -856,7 +857,7 @@ def retrieval():
     objs = APIToken.query(token=token)
     if not objs:
         return get_json_result(
-            data=False, message='Authentication error: API key is invalid!"', code=settings.RetCode.AUTHENTICATION_ERROR)
+            data=False, message='Authentication error: API key is invalid!"', code=RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
     kb_ids = req.get("kb_id", [])
@@ -867,7 +868,7 @@ def retrieval():
     similarity_threshold = float(req.get("similarity_threshold", 0.2))
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
     top = int(req.get("top_k", 1024))
-    highlight = bool(req.get("highlight", False)) 
+    highlight = bool(req.get("highlight", False))
 
     try:
         kbs = KnowledgebaseService.get_by_ids(kb_ids)
@@ -875,7 +876,7 @@ def retrieval():
         if len(embd_nms) != 1:
             return get_json_result(
                 data=False, message='Knowledge bases use different embedding models or does not exist."',
-                code=settings.RetCode.AUTHENTICATION_ERROR)
+                code=RetCode.AUTHENTICATION_ERROR)
 
         embd_mdl = LLMBundle(kbs[0].tenant_id, LLMType.EMBEDDING, llm_name=kbs[0].embd_id)
         rerank_mdl = None
@@ -894,5 +895,5 @@ def retrieval():
     except Exception as e:
         if str(e).find("not_found") > 0:
             return get_json_result(data=False, message='No chunk found! Check the chunk status please!',
-                                   code=settings.RetCode.DATA_ERROR)
+                                   code=RetCode.DATA_ERROR)
         return server_error_response(e)

api/apps/canvas_app.py

@@ -25,7 +25,6 @@ from flask import request, Response
 from flask_login import login_required, current_user
 
 from agent.component import LLM
-from api import settings
 from api.db import CanvasCategory, FileType
 from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService, API4ConversationService
 from api.db.services.document_service import DocumentService
@@ -34,8 +33,8 @@ from api.db.services.pipeline_operation_log_service import PipelineOperationLogS
 from api.db.services.task_service import queue_dataflow, CANVAS_DEBUG_DOC_ID, TaskService
 from api.db.services.user_service import TenantService
 from api.db.services.user_canvas_version import UserCanvasVersionService
-from api.settings import RetCode
-from api.utils import get_uuid
+from common.constants import RetCode
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result
 from agent.canvas import Canvas
 from peewee import MySQLDatabase, PostgresqlDatabase
@@ -46,6 +45,7 @@ from api.utils.file_utils import filename_type, read_potential_broken_pdf
 from rag.flow.pipeline import Pipeline
 from rag.nlp import search
 from rag.utils.redis_conn import REDIS_CONN
+from common import settings
 
 
 @manager.route('/templates', methods=['GET'])  # noqa: F821

api/apps/chunk_app.py

@@ -21,8 +21,6 @@ import xxhash
 from flask import request
 from flask_login import current_user, login_required
 
-from api import settings
-from api.db import LLMType, ParserType
 from api.db.services.dialog_service import meta_filter
 from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
@@ -34,8 +32,9 @@ from rag.app.qa import beAdoc, rmPrefix
 from rag.app.tag import label_question
 from rag.nlp import rag_tokenizer, search
 from rag.prompts.generator import gen_meta_filter, cross_languages, keyword_extraction
-from rag.settings import PAGERANK_FLD
-from rag.utils import rmSpace
+from common.string_utils import remove_redundant_spaces
+from common.constants import RetCode, LLMType, ParserType, PAGERANK_FLD
+from common import settings
 
 
 @manager.route('/list', methods=['POST'])  # noqa: F821
@@ -65,7 +64,7 @@ def list_chunk():
         for id in sres.ids:
             d = {
                 "chunk_id": id,
-                "content_with_weight": rmSpace(sres.highlight[id]) if question and id in sres.highlight else sres.field[
+                "content_with_weight": remove_redundant_spaces(sres.highlight[id]) if question and id in sres.highlight else sres.field[
                     id].get(
                     "content_with_weight", ""),
                 "doc_id": sres.field[id]["doc_id"],
@@ -83,7 +82,7 @@ def list_chunk():
     except Exception as e:
         if str(e).find("not_found") > 0:
             return get_json_result(data=False, message='No chunk found!',
-                                   code=settings.RetCode.DATA_ERROR)
+                                   code=RetCode.DATA_ERROR)
         return server_error_response(e)
 
 
@@ -115,7 +114,7 @@ def get():
     except Exception as e:
         if str(e).find("NotFoundError") >= 0:
             return get_json_result(data=False, message='Chunk not found!',
-                                   code=settings.RetCode.DATA_ERROR)
+                                   code=RetCode.DATA_ERROR)
         return server_error_response(e)
 
 
@@ -200,7 +199,6 @@ def switch():
 @login_required
 @validate_request("chunk_ids", "doc_id")
 def rm():
-    from rag.utils.storage_factory import STORAGE_IMPL
     req = request.json
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
@@ -214,8 +212,8 @@ def rm():
         chunk_number = len(deleted_chunk_ids)
         DocumentService.decrement_chunk_num(doc.id, doc.kb_id, 1, chunk_number, 0)
         for cid in deleted_chunk_ids:
-            if STORAGE_IMPL.obj_exist(doc.kb_id, cid):
-                STORAGE_IMPL.rm(doc.kb_id, cid)
+            if settings.STORAGE_IMPL.obj_exist(doc.kb_id, cid):
+                settings.STORAGE_IMPL.rm(doc.kb_id, cid)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
@@ -292,7 +290,7 @@ def retrieval_test():
         kb_ids = [kb_ids]
     if not kb_ids:
         return get_json_result(data=False, message='Please specify dataset firstly.',
-                               code=settings.RetCode.DATA_ERROR)
+                               code=RetCode.DATA_ERROR)
 
     doc_ids = req.get("doc_ids", [])
     use_kg = req.get("use_kg", False)
@@ -326,7 +324,7 @@ def retrieval_test():
             else:
                 return get_json_result(
                     data=False, message='Only owner of knowledgebase authorized for this operation.',
-                    code=settings.RetCode.OPERATING_ERROR)
+                    code=RetCode.OPERATING_ERROR)
 
         e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
         if not e:
@@ -371,7 +369,7 @@ def retrieval_test():
     except Exception as e:
         if str(e).find("not_found") > 0:
             return get_json_result(data=False, message='No chunk found! Check the chunk status please!',
-                                   code=settings.RetCode.DATA_ERROR)
+                                   code=RetCode.DATA_ERROR)
         return server_error_response(e)
 
 

api/apps/connector_app.py

@@ -0,0 +1,106 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import time
+
+from flask import request
+from flask_login import login_required, current_user
+
+from api.db import InputType
+from api.db.services.connector_service import ConnectorService, Connector2KbService, SyncLogsService
+from api.utils.api_utils import get_json_result, validate_request, get_data_error_result
+from common.misc_utils import get_uuid
+from common.constants import RetCode, TaskStatus
+
+@manager.route("/set", methods=["POST"])  # noqa: F821
+@login_required
+def set_connector():
+    req = request.json
+    if req.get("id"):
+        conn = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req}
+        ConnectorService.update_by_id(req["id"], conn)
+    else:
+        req["id"] = get_uuid()
+        conn = {
+            "id": req["id"],
+            "tenant_id": current_user.id,
+            "name": req["name"],
+            "source": req["source"],
+            "input_type": InputType.POLL,
+            "config": req["config"],
+            "refresh_freq": int(req.get("refresh_freq", 30)),
+            "prune_freq": int(req.get("prune_freq", 720)),
+            "timeout_secs": int(req.get("timeout_secs", 60*29)),
+            "status": TaskStatus.SCHEDULE
+        }
+        conn["status"] = TaskStatus.SCHEDULE
+        ConnectorService.save(**conn)
+
+    time.sleep(1)
+    e, conn = ConnectorService.get_by_id(req["id"])
+
+    return get_json_result(data=conn.to_dict())
+
+
+@manager.route("/list", methods=["GET"])  # noqa: F821
+@login_required
+def list_connector():
+    return get_json_result(data=ConnectorService.list(current_user.id))
+
+
+@manager.route("/<connector_id>", methods=["GET"])  # noqa: F821
+@login_required
+def get_connector(connector_id):
+    e, conn = ConnectorService.get_by_id(connector_id)
+    if not e:
+        return get_data_error_result(message="Can't find this Connector!")
+    return get_json_result(data=conn.to_dict())
+
+
+@manager.route("/<connector_id>/logs", methods=["GET"])  # noqa: F821
+@login_required
+def list_logs(connector_id):
+    req = request.args.to_dict(flat=True)
+    return get_json_result(data=SyncLogsService.list_sync_tasks(connector_id, int(req.get("page", 1)), int(req.get("page_size", 15))))
+
+
+@manager.route("/<connector_id>/resume", methods=["PUT"])  # noqa: F821
+@login_required
+def resume(connector_id):
+    req = request.json
+    if req.get("resume"):
+        ConnectorService.resume(connector_id, TaskStatus.SCHEDULE)
+    else:
+        ConnectorService.resume(connector_id, TaskStatus.CANCEL)
+    return get_json_result(data=True)
+
+
+@manager.route("/<connector_id>/link", methods=["POST"])  # noqa: F821
+@validate_request("kb_ids")
+@login_required
+def link_kb(connector_id):
+    req = request.json
+    errors = Connector2KbService.link_kb(connector_id, req["kb_ids"], current_user.id)
+    if errors:
+        return get_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
+    return get_json_result(data=True)
+
+
+@manager.route("/<connector_id>/rm", methods=["POST"])  # noqa: F821
+@login_required
+def rm_connector(connector_id):
+    ConnectorService.resume(connector_id, TaskStatus.CANCEL)
+    ConnectorService.delete_by_id(connector_id)
+    return get_json_result(data=True)

api/apps/conversation_app.py

@@ -19,8 +19,6 @@ import logging
 from copy import deepcopy
 from flask import Response, request
 from flask_login import current_user, login_required
-from api import settings
-from api.db import LLMType
 from api.db.db_models import APIToken
 from api.db.services.conversation_service import ConversationService, structure_answer
 from api.db.services.dialog_service import DialogService, ask, chat, gen_mindmap
@@ -31,6 +29,7 @@ from api.db.services.user_service import TenantService, UserTenantService
 from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request
 from rag.prompts.template import load_prompt
 from rag.prompts.generator import chunks_format
+from common.constants import RetCode, LLMType
 
 
 @manager.route("/set", methods=["POST"])  # noqa: F821
@@ -93,7 +92,7 @@ def get():
                 avatar = dialog[0].icon
                 break
         else:
-            return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+            return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=RetCode.OPERATING_ERROR)
 
         for ref in conv.reference:
             if isinstance(ref, list):
@@ -142,7 +141,7 @@ def rm():
                 if DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id):
                     break
             else:
-                return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+                return get_json_result(data=False, message="Only owner of conversation authorized for this operation.", code=RetCode.OPERATING_ERROR)
             ConversationService.delete_by_id(cid)
         return get_json_result(data=True)
     except Exception as e:
@@ -155,7 +154,7 @@ def list_conversation():
     dialog_id = request.args["dialog_id"]
     try:
         if not DialogService.query(tenant_id=current_user.id, id=dialog_id):
-            return get_json_result(data=False, message="Only owner of dialog authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+            return get_json_result(data=False, message="Only owner of dialog authorized for this operation.", code=RetCode.OPERATING_ERROR)
         convs = ConversationService.query(dialog_id=dialog_id, order_by=ConversationService.model.create_time, reverse=True)
 
         convs = [d.to_dict() for d in convs]

api/apps/dialog_app.py

@@ -18,13 +18,13 @@ from flask import request
 from flask_login import login_required, current_user
 from api.db.services import duplicate_name
 from api.db.services.dialog_service import DialogService
-from api.db import StatusEnum
+from common.constants import StatusEnum
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import TenantService, UserTenantService
-from api import settings
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
+from common.constants import RetCode
 from api.utils.api_utils import get_json_result
 
 
@@ -219,7 +219,7 @@ def rm():
             else:
                 return get_json_result(
                     data=False, message='Only owner of dialog authorized for this operation.',
-                    code=settings.RetCode.OPERATING_ERROR)
+                    code=RetCode.OPERATING_ERROR)
             dialog_list.append({"id": id,"status":StatusEnum.INVALID.value})
         DialogService.update_many_by_id(dialog_list)
         return get_json_result(data=True)

api/apps/document_app.py

@@ -23,30 +23,31 @@ import flask
 from flask import request
 from flask_login import current_user, login_required
 
-from api import settings
 from api.common.check_team_permission import check_kb_team_permission
 from api.constants import FILE_NAME_LEN_LIMIT, IMG_BASE64_PREFIX
-from api.db import VALID_FILE_TYPES, VALID_TASK_STATUS, FileSource, FileType, ParserType, TaskStatus
-from api.db.db_models import File, Task
+from api.db import VALID_FILE_TYPES, FileType
+from api.db.db_models import Task
 from api.db.services import duplicate_name
 from api.db.services.document_service import DocumentService, doc_upload_and_parse
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.task_service import TaskService, cancel_all_task_of, queue_tasks, queue_dataflow
+from api.db.services.task_service import TaskService, cancel_all_task_of
 from api.db.services.user_service import UserTenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import (
     get_data_error_result,
     get_json_result,
     server_error_response,
     validate_request,
 )
-from api.utils.file_utils import filename_type, get_project_base_directory, thumbnail
+from api.utils.file_utils import filename_type, thumbnail
+from common.file_utils import get_project_base_directory
+from common.constants import RetCode, VALID_TASK_STATUS, ParserType, TaskStatus
 from api.utils.web_utils import CONTENT_TYPE_MAP, html2pdf, is_valid_url
 from deepdoc.parser.html_parser import RAGFlowHtmlParser
 from rag.nlp import search, rag_tokenizer
-from rag.utils.storage_factory import STORAGE_IMPL
+from common import settings
 
 
 @manager.route("/upload", methods=["POST"])  # noqa: F821
@@ -55,29 +56,29 @@ from rag.utils.storage_factory import STORAGE_IMPL
 def upload():
     kb_id = request.form.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     if "file" not in request.files:
-        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
 
     file_objs = request.files.getlist("file")
     for file_obj in file_objs:
         if file_obj.filename == "":
-            return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR)
         if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
-            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
 
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
     if not check_kb_team_permission(kb, current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
     err, files = FileService.upload_document(kb, file_objs, current_user.id)
     if err:
-        return get_json_result(data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
+        return get_json_result(data=files, message="\n".join(err), code=RetCode.SERVER_ERROR)
 
     if not files:
-        return get_json_result(data=files, message="There seems to be an issue with your file format. Please verify it is correct and not corrupted.", code=settings.RetCode.DATA_ERROR)
+        return get_json_result(data=files, message="There seems to be an issue with your file format. Please verify it is correct and not corrupted.", code=RetCode.DATA_ERROR)
     files = [f[0] for f in files]  # remove the blob
 
     return get_json_result(data=files)
@@ -89,16 +90,16 @@ def upload():
 def web_crawl():
     kb_id = request.form.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     name = request.form.get("name")
     url = request.form.get("url")
     if not is_valid_url(url):
-        return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
         raise LookupError("Can't find this knowledgebase!")
     if check_kb_team_permission(kb, current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
     blob = html2pdf(url)
     if not blob:
@@ -117,9 +118,9 @@ def web_crawl():
             raise RuntimeError("This type of file has not been supported yet!")
 
         location = filename
-        while STORAGE_IMPL.obj_exist(kb_id, location):
+        while settings.STORAGE_IMPL.obj_exist(kb_id, location):
             location += "_"
-        STORAGE_IMPL.put(kb_id, location, blob)
+        settings.STORAGE_IMPL.put(kb_id, location, blob)
         doc = {
             "id": get_uuid(),
             "kb_id": kb.id,
@@ -155,12 +156,12 @@ def create():
     req = request.json
     kb_id = req["kb_id"]
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
-        return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
 
     if req["name"].strip() == "":
-        return get_json_result(data=False, message="File name can't be empty.", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="File name can't be empty.", code=RetCode.ARGUMENT_ERROR)
     req["name"] = req["name"].strip()
 
     try:
@@ -210,13 +211,13 @@ def create():
 def list_docs():
     kb_id = request.args.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     tenants = UserTenantService.query(user_id=current_user.id)
     for tenant in tenants:
         if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
             break
     else:
-        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
     keywords = request.args.get("keywords", "")
 
     page_number = int(request.args.get("page", 0))
@@ -272,13 +273,13 @@ def get_filter():
 
     kb_id = req.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
     tenants = UserTenantService.query(user_id=current_user.id)
     for tenant in tenants:
         if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
             break
     else:
-        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=settings.RetCode.OPERATING_ERROR)
+        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
 
     keywords = req.get("keywords", "")
 
@@ -310,7 +311,7 @@ def docinfos():
     doc_ids = req["doc_ids"]
     for doc_id in doc_ids:
         if not DocumentService.accessible(doc_id, current_user.id):
-            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     docs = DocumentService.get_by_ids(doc_ids)
     return get_json_result(data=list(docs.dicts()))
 
@@ -320,7 +321,7 @@ def docinfos():
 def thumbnails():
     doc_ids = request.args.getlist("doc_ids")
     if not doc_ids:
-        return get_json_result(data=False, message='Lack of "Document ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "Document ID"', code=RetCode.ARGUMENT_ERROR)
 
     try:
         docs = DocumentService.get_thumbnails(doc_ids)
@@ -343,7 +344,7 @@ def change_status():
     status = str(req.get("status", ""))
 
     if status not in ["0", "1"]:
-        return get_json_result(data=False, message='"Status" must be either 0 or 1!', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='"Status" must be either 0 or 1!', code=RetCode.ARGUMENT_ERROR)
 
     result = {}
     for doc_id in doc_ids:
@@ -385,50 +386,12 @@ def rm():
 
     for doc_id in doc_ids:
         if not DocumentService.accessible4deletion(doc_id, current_user.id):
-            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
-    root_folder = FileService.get_root_folder(current_user.id)
-    pf_id = root_folder["id"]
-    FileService.init_knowledgebase_docs(pf_id, current_user.id)
-    errors = ""
-    kb_table_num_map = {}
-    for doc_id in doc_ids:
-        try:
-            e, doc = DocumentService.get_by_id(doc_id)
-            if not e:
-                return get_data_error_result(message="Document not found!")
-            tenant_id = DocumentService.get_tenant_id(doc_id)
-            if not tenant_id:
-                return get_data_error_result(message="Tenant not found!")
-
-            b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
-
-            TaskService.filter_delete([Task.doc_id == doc_id])
-            if not DocumentService.remove_document(doc, tenant_id):
-                return get_data_error_result(message="Database error (Document removal)!")
-
-            f2d = File2DocumentService.get_by_document_id(doc_id)
-            deleted_file_count = 0
-            if f2d:
-                deleted_file_count = FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
-            File2DocumentService.delete_by_document_id(doc_id)
-            if deleted_file_count > 0:
-                STORAGE_IMPL.rm(b, n)
-
-            doc_parser = doc.parser_id
-            if doc_parser == ParserType.TABLE:
-                kb_id = doc.kb_id
-                if kb_id not in kb_table_num_map:
-                    counts = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
-                    kb_table_num_map[kb_id] = counts
-                kb_table_num_map[kb_id] -= 1
-                if kb_table_num_map[kb_id] <= 0:
-                    KnowledgebaseService.delete_field_map(kb_id)
-        except Exception as e:
-            errors += str(e)
+    errors = FileService.delete_docs(doc_ids, current_user.id)
 
     if errors:
-        return get_json_result(data=False, message=errors, code=settings.RetCode.SERVER_ERROR)
+        return get_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
 
     return get_json_result(data=True)
 
@@ -440,7 +403,7 @@ def run():
     req = request.json
     for doc_id in req["doc_ids"]:
         if not DocumentService.accessible(doc_id, current_user.id):
-            return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         kb_table_num_map = {}
         for id in req["doc_ids"]:
@@ -473,23 +436,7 @@ def run():
 
             if str(req["run"]) == TaskStatus.RUNNING.value:
                 doc = doc.to_dict()
-                doc["tenant_id"] = tenant_id
-
-                doc_parser = doc.get("parser_id", ParserType.NAIVE)
-                if doc_parser == ParserType.TABLE:
-                    kb_id = doc.get("kb_id")
-                    if not kb_id:
-                        continue
-                    if kb_id not in kb_table_num_map:
-                        count = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
-                        kb_table_num_map[kb_id] = count
-                        if kb_table_num_map[kb_id] <= 0:
-                            KnowledgebaseService.delete_field_map(kb_id)
-                if doc.get("pipeline_id", ""):
-                    queue_dataflow(tenant_id, flow_id=doc["pipeline_id"], task_id=get_uuid(), doc_id=id)
-                else:
-                    bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
-                    queue_tasks(doc, bucket, name, 0)
+                DocumentService.run(tenant_id, doc, kb_table_num_map)
 
         return get_json_result(data=True)
     except Exception as e:
@@ -502,15 +449,15 @@ def run():
 def rename():
     req = request.json
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
             return get_data_error_result(message="Document not found!")
         if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
-            return get_json_result(data=False, message="The extension of file can't be changed", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="The extension of file can't be changed", code=RetCode.ARGUMENT_ERROR)
         if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
-            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
 
         for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
             if d.name == req["name"]:
@@ -553,7 +500,7 @@ def get(doc_id):
             return get_data_error_result(message="Document not found!")
 
         b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
-        response = flask.make_response(STORAGE_IMPL.get(b, n))
+        response = flask.make_response(settings.STORAGE_IMPL.get(b, n))
 
         ext = re.search(r"\.([^.]+)$", doc.name.lower())
         ext = ext.group(1) if ext else None
@@ -575,7 +522,7 @@ def change_parser():
 
     req = request.json
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
     e, doc = DocumentService.get_by_id(req["doc_id"])
     if not e:
@@ -629,7 +576,7 @@ def get_image(image_id):
         if len(arr) != 2:
             return get_data_error_result(message="Image not found.")
         bkt, nm = image_id.split("-")
-        response = flask.make_response(STORAGE_IMPL.get(bkt, nm))
+        response = flask.make_response(settings.STORAGE_IMPL.get(bkt, nm))
         response.headers.set("Content-Type", "image/JPEG")
         return response
     except Exception as e:
@@ -641,12 +588,12 @@ def get_image(image_id):
 @validate_request("conversation_id")
 def upload_and_parse():
     if "file" not in request.files:
-        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
 
     file_objs = request.files.getlist("file")
     for file_obj in file_objs:
         if file_obj.filename == "":
-            return get_json_result(data=False, message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR)
 
     doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, current_user.id)
 
@@ -659,7 +606,7 @@ def parse():
     url = request.json.get("url") if request.json else ""
     if url:
         if not is_valid_url(url):
-            return get_json_result(data=False, message="The URL format is invalid", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
         download_path = os.path.join(get_project_base_directory(), "logs/downloads")
         os.makedirs(download_path, exist_ok=True)
         from seleniumwire.webdriver import Chrome, ChromeOptions
@@ -692,13 +639,13 @@ def parse():
 
         r = re.search(r"filename=\"([^\"]+)\"", str(res_headers))
         if not r or not r.group(1):
-            return get_json_result(data=False, message="Can't not identify downloaded file", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="Can't not identify downloaded file", code=RetCode.ARGUMENT_ERROR)
         f = File(r.group(1), os.path.join(download_path, r.group(1)))
         txt = FileService.parse_docs([f], current_user.id)
         return get_json_result(data=txt)
 
     if "file" not in request.files:
-        return get_json_result(data=False, message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
 
     file_objs = request.files.getlist("file")
     txt = FileService.parse_docs(file_objs, current_user.id)
@@ -712,18 +659,18 @@ def parse():
 def set_meta():
     req = request.json
     if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         meta = json.loads(req["meta"])
         if not isinstance(meta, dict):
-            return get_json_result(data=False, message="Only dictionary type supported.", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_json_result(data=False, message="Only dictionary type supported.", code=RetCode.ARGUMENT_ERROR)
         for k, v in meta.items():
             if not isinstance(v, str) and not isinstance(v, int) and not isinstance(v, float):
-                return get_json_result(data=False, message=f"The type is not supported: {v}", code=settings.RetCode.ARGUMENT_ERROR)
+                return get_json_result(data=False, message=f"The type is not supported: {v}", code=RetCode.ARGUMENT_ERROR)
     except Exception as e:
-        return get_json_result(data=False, message=f"Json syntax error: {e}", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message=f"Json syntax error: {e}", code=RetCode.ARGUMENT_ERROR)
     if not isinstance(meta, dict):
-        return get_json_result(data=False, message='Meta data should be in Json map format, like {"key": "value"}', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Meta data should be in Json map format, like {"key": "value"}', code=RetCode.ARGUMENT_ERROR)
 
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])

api/apps/file2document_app.py

@@ -23,10 +23,10 @@ from flask import request
 from flask_login import login_required, current_user
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
+from common.constants import RetCode
 from api.db import FileType
 from api.db.services.document_service import DocumentService
-from api import settings
 from api.utils.api_utils import get_json_result
 
 
@@ -108,7 +108,7 @@ def rm():
     file_ids = req["file_ids"]
     if not file_ids:
         return get_json_result(
-            data=False, message='Lack of "Files ID"', code=settings.RetCode.ARGUMENT_ERROR)
+            data=False, message='Lack of "Files ID"', code=RetCode.ARGUMENT_ERROR)
     try:
         for file_id in file_ids:
             informs = File2DocumentService.get_by_file_id(file_id)

api/apps/file_app.py

@@ -26,15 +26,15 @@ from api.common.check_team_permission import check_file_team_permission
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.utils import get_uuid
-from api.db import FileType, FileSource
+from common.misc_utils import get_uuid
+from common.constants import RetCode, FileSource
+from api.db import FileType
 from api.db.services import duplicate_name
 from api.db.services.file_service import FileService
-from api import settings
 from api.utils.api_utils import get_json_result
 from api.utils.file_utils import filename_type
 from api.utils.web_utils import CONTENT_TYPE_MAP
-from rag.utils.storage_factory import STORAGE_IMPL
+from common import settings
 
 
 @manager.route('/upload', methods=['POST'])  # noqa: F821
@@ -49,21 +49,21 @@ def upload():
 
     if 'file' not in request.files:
         return get_json_result(
-            data=False, message='No file part!', code=settings.RetCode.ARGUMENT_ERROR)
+            data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
     file_objs = request.files.getlist('file')
 
     for file_obj in file_objs:
         if file_obj.filename == '':
             return get_json_result(
-                data=False, message='No file selected!', code=settings.RetCode.ARGUMENT_ERROR)
+                data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR)
     file_res = []
     try:
         e, pf_folder = FileService.get_by_id(pf_id)
         if not e:
             return get_data_error_result( message="Can't find this folder!")
         for file_obj in file_objs:
-            MAX_FILE_NUM_PER_USER = int(os.environ.get('MAX_FILE_NUM_PER_USER', 0))
-            if MAX_FILE_NUM_PER_USER > 0 and DocumentService.get_doc_count(current_user.id) >= MAX_FILE_NUM_PER_USER:
+            MAX_FILE_NUM_PER_USER: int = int(os.environ.get('MAX_FILE_NUM_PER_USER', 0))
+            if 0 < MAX_FILE_NUM_PER_USER <= DocumentService.get_doc_count(current_user.id):
                 return get_data_error_result( message="Exceed the maximum file number of a free user!")
 
             # split file name path
@@ -95,14 +95,14 @@ def upload():
             # file type
             filetype = filename_type(file_obj_names[file_len - 1])
             location = file_obj_names[file_len - 1]
-            while STORAGE_IMPL.obj_exist(last_folder.id, location):
+            while settings.STORAGE_IMPL.obj_exist(last_folder.id, location):
                 location += "_"
             blob = file_obj.read()
             filename = duplicate_name(
                 FileService.query,
                 name=file_obj_names[file_len - 1],
                 parent_id=last_folder.id)
-            STORAGE_IMPL.put(last_folder.id, location, blob)
+            settings.STORAGE_IMPL.put(last_folder.id, location, blob)
             file = {
                 "id": get_uuid(),
                 "parent_id": last_folder.id,
@@ -134,7 +134,7 @@ def create():
     try:
         if not FileService.is_parent_folder_exist(pf_id):
             return get_json_result(
-                data=False, message="Parent Folder Doesn't Exist!", code=settings.RetCode.OPERATING_ERROR)
+                data=False, message="Parent Folder Doesn't Exist!", code=RetCode.OPERATING_ERROR)
         if FileService.query(name=req["name"], parent_id=pf_id):
             return get_data_error_result(
                 message="Duplicated folder name in the same folder.")
@@ -245,7 +245,7 @@ def rm():
     def _delete_single_file(file):
         try:
             if file.location:
-                STORAGE_IMPL.rm(file.parent_id, file.location)
+                settings.STORAGE_IMPL.rm(file.parent_id, file.location)
         except Exception:
             logging.exception(f"Fail to remove object: {file.parent_id}/{file.location}")
 
@@ -279,7 +279,7 @@ def rm():
             if not file.tenant_id:
                 return get_data_error_result(message="Tenant not found!")
             if not check_file_team_permission(file, current_user.id):
-                return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+                return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
             if file.source_type == FileSource.KNOWLEDGEBASE:
                 continue
@@ -306,14 +306,14 @@ def rename():
         if not e:
             return get_data_error_result(message="File not found!")
         if not check_file_team_permission(file, current_user.id):
-            return get_json_result(data=False, message='No authorization.', code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_json_result(data=False, message='No authorization.', code=RetCode.AUTHENTICATION_ERROR)
         if file.type != FileType.FOLDER.value \
             and pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
                 file.name.lower()).suffix:
             return get_json_result(
                 data=False,
                 message="The extension of file can't be changed",
-                code=settings.RetCode.ARGUMENT_ERROR)
+                code=RetCode.ARGUMENT_ERROR)
         for file in FileService.query(name=req["name"], pf_id=file.parent_id):
             if file.name == req["name"]:
                 return get_data_error_result(
@@ -344,12 +344,12 @@ def get(file_id):
         if not e:
             return get_data_error_result(message="Document not found!")
         if not check_file_team_permission(file, current_user.id):
-            return get_json_result(data=False, message='No authorization.', code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_json_result(data=False, message='No authorization.', code=RetCode.AUTHENTICATION_ERROR)
 
-        blob = STORAGE_IMPL.get(file.parent_id, file.location)
+        blob = settings.STORAGE_IMPL.get(file.parent_id, file.location)
         if not blob:
             b, n = File2DocumentService.get_storage_address(file_id=file_id)
-            blob = STORAGE_IMPL.get(b, n)
+            blob = settings.STORAGE_IMPL.get(b, n)
 
         response = flask.make_response(blob)
         ext = re.search(r"\.([^.]+)$", file.name.lower())
@@ -376,7 +376,7 @@ def move():
 
         ok, dest_folder = FileService.get_by_id(dest_parent_id)
         if not ok or not dest_folder:
-            return get_data_error_result(message="Parent Folder not found!")
+            return get_data_error_result(message="Parent folder not found!")
 
         files = FileService.get_by_ids(file_ids)
         if not files:
@@ -387,14 +387,14 @@ def move():
         for file_id in file_ids:
             file = files_dict.get(file_id)
             if not file:
-                return get_data_error_result(message="File or Folder not found!")
+                return get_data_error_result(message="File or folder not found!")
             if not file.tenant_id:
                 return get_data_error_result(message="Tenant not found!")
             if not check_file_team_permission(file, current_user.id):
                 return get_json_result(
                     data=False,
                     message="No authorization.",
-                    code=settings.RetCode.AUTHENTICATION_ERROR,
+                    code=RetCode.AUTHENTICATION_ERROR,
                 )
 
         def _move_entry_recursive(source_file_entry, dest_folder):
@@ -428,11 +428,11 @@ def move():
             filename = source_file_entry.name
 
             new_location = filename
-            while STORAGE_IMPL.obj_exist(dest_folder.id, new_location):
+            while settings.STORAGE_IMPL.obj_exist(dest_folder.id, new_location):
                 new_location += "_"
 
             try:
-                STORAGE_IMPL.move(old_parent_id, old_location, dest_folder.id, new_location)
+                settings.STORAGE_IMPL.move(old_parent_id, old_location, dest_folder.id, new_location)
             except Exception as storage_err:
                 raise RuntimeError(f"Move file failed at storage layer: {str(storage_err)}")
 

api/apps/kb_app.py

@@ -15,11 +15,15 @@
 #
 import json
 import logging
+import random
 
 from flask import request
 from flask_login import login_required, current_user
+import numpy as np
 
-from api.db.services import duplicate_name
+
+from api.db.services.connector_service import Connector2KbService
+from api.db.services.llm_service import LLMBundle
 from api.db.services.document_service import DocumentService, queue_raptor_o_graphrag_tasks
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
@@ -27,81 +31,33 @@ from api.db.services.pipeline_operation_log_service import PipelineOperationLogS
 from api.db.services.task_service import TaskService, GRAPH_RAPTOR_FAKE_DOC_ID
 from api.db.services.user_service import TenantService, UserTenantService
 from api.utils.api_utils import get_error_data_result, server_error_response, get_data_error_result, validate_request, not_allowed_parameters
-from api.utils import get_uuid
-from api.db import PipelineTaskType, StatusEnum, FileSource, VALID_FILE_TYPES, VALID_TASK_STATUS
+from api.db import VALID_FILE_TYPES
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.db_models import File
 from api.utils.api_utils import get_json_result
-from api import settings
 from rag.nlp import search
 from api.constants import DATASET_NAME_LIMIT
-from rag.settings import PAGERANK_FLD
 from rag.utils.redis_conn import REDIS_CONN
-from rag.utils.storage_factory import STORAGE_IMPL
-
+from rag.utils.doc_store_conn import OrderByExpr  
+from common.constants import RetCode, PipelineTaskType, StatusEnum, VALID_TASK_STATUS, FileSource, LLMType, PAGERANK_FLD
+from common import settings
 
 @manager.route('/create', methods=['post'])  # noqa: F821
 @login_required
 @validate_request("name")
 def create():
     req = request.json
-    dataset_name = req["name"]
-    if not isinstance(dataset_name, str):
-        return get_data_error_result(message="Dataset name must be string.")
-    if dataset_name.strip() == "":
-        return get_data_error_result(message="Dataset name can't be empty.")
-    if len(dataset_name.encode("utf-8")) > DATASET_NAME_LIMIT:
-        return get_data_error_result(
-            message=f"Dataset name length is {len(dataset_name)} which is larger than {DATASET_NAME_LIMIT}")
+    req = KnowledgebaseService.create_with_name(
+        name = req.pop("name", None),
+        tenant_id = current_user.id,
+        parser_id = req.pop("parser_id", None),
+        **req
+    )        
 
-    dataset_name = dataset_name.strip()
-    dataset_name = duplicate_name(
-        KnowledgebaseService.query,
-        name=dataset_name,
-        tenant_id=current_user.id,
-        status=StatusEnum.VALID.value)
     try:
-        req["id"] = get_uuid()
-        req["name"] = dataset_name
-        req["tenant_id"] = current_user.id
-        req["created_by"] = current_user.id
-        if not req.get("parser_id"):
-            req["parser_id"] = "naive"
-        e, t = TenantService.get_by_id(current_user.id)
-        if not e:
-            return get_data_error_result(message="Tenant not found.")
-
-        req["parser_config"] = {
-            "layout_recognize": "DeepDOC",
-            "chunk_token_num": 512,
-            "delimiter": "\n",
-            "auto_keywords": 0,
-            "auto_questions": 0,
-            "html4excel": False,
-            "topn_tags": 3,
-            "raptor": {
-                "use_raptor": True,
-                "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n      {cluster_content}\nThe above is the content you need to summarize.",
-                "max_token": 256,
-                "threshold": 0.1,
-                "max_cluster": 64,
-                "random_seed": 0
-            },
-            "graphrag": {
-                "use_graphrag": True,
-                "entity_types": [
-                    "organization",
-                    "person",
-                    "geo",
-                    "event",
-                    "category"
-                ],
-                "method": "light"
-            }
-        }
         if not KnowledgebaseService.save(**req):
             return get_data_error_result()
-        return get_json_result(data={"kb_id": req["id"]})
+        return get_json_result(data={"kb_id":req["id"]})
     except Exception as e:
         return server_error_response(e)
 
@@ -125,14 +81,14 @@ def update():
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
     try:
         if not KnowledgebaseService.query(
                 created_by=current_user.id, id=req["kb_id"]):
             return get_json_result(
                 data=False, message='Only owner of knowledgebase authorized for this operation.',
-                code=settings.RetCode.OPERATING_ERROR)
+                code=RetCode.OPERATING_ERROR)
 
         e, kb = KnowledgebaseService.get_by_id(req["kb_id"])
         if not e:
@@ -146,6 +102,10 @@ def update():
                 message="Duplicated knowledgebase name.")
 
         del req["kb_id"]
+        connectors = []
+        if "connectors" in req:
+            connectors = req["connectors"]
+            del req["connectors"]
         if not KnowledgebaseService.update_by_id(kb.id, req):
             return get_data_error_result()
 
@@ -162,6 +122,10 @@ def update():
         if not e:
             return get_data_error_result(
                 message="Database error (Knowledgebase rename)!")
+        if connectors:
+            errors = Connector2KbService.link_connectors(kb.id, [conn["id"] for conn in connectors], current_user.id)
+            if errors:
+                logging.error("Link KB errors: ", errors)
         kb = kb.to_dict()
         kb.update(req)
 
@@ -183,12 +147,14 @@ def detail():
         else:
             return get_json_result(
                 data=False, message='Only owner of knowledgebase authorized for this operation.',
-                code=settings.RetCode.OPERATING_ERROR)
+                code=RetCode.OPERATING_ERROR)
         kb = KnowledgebaseService.get_detail(kb_id)
         if not kb:
             return get_data_error_result(
                 message="Can't find this knowledgebase!")
         kb["size"] = DocumentService.get_total_size_by_kb_id(kb_id=kb["id"],keywords="", run_status=[], types=[])
+        kb["connectors"] = Connector2KbService.list_connectors(kb_id)
+
         for key in ["graphrag_task_finish_at", "raptor_task_finish_at", "mindmap_task_finish_at"]:
             if finish_at := kb.get(key):
                 kb[key] = finish_at.strftime("%Y-%m-%d %H:%M:%S")
@@ -241,7 +207,7 @@ def rm():
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
     try:
         kbs = KnowledgebaseService.query(
@@ -249,7 +215,7 @@ def rm():
         if not kbs:
             return get_json_result(
                 data=False, message='Only owner of knowledgebase authorized for this operation.',
-                code=settings.RetCode.OPERATING_ERROR)
+                code=RetCode.OPERATING_ERROR)
 
         for doc in DocumentService.query(kb_id=req["kb_id"]):
             if not DocumentService.remove_document(doc, kbs[0].tenant_id):
@@ -267,8 +233,8 @@ def rm():
         for kb in kbs:
             settings.docStoreConn.delete({"kb_id": kb.id}, search.index_name(kb.tenant_id), kb.id)
             settings.docStoreConn.deleteIdx(search.index_name(kb.tenant_id), kb.id)
-            if hasattr(STORAGE_IMPL, 'remove_bucket'):
-                STORAGE_IMPL.remove_bucket(kb.id)
+            if hasattr(settings.STORAGE_IMPL, 'remove_bucket'):
+                settings.STORAGE_IMPL.remove_bucket(kb.id)
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
@@ -281,7 +247,7 @@ def list_tags(kb_id):
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
 
     tenants = UserTenantService.get_tenants_by_user_id(current_user.id)
@@ -300,7 +266,7 @@ def list_tags_from_kbs():
             return get_json_result(
                 data=False,
                 message='No authorization.',
-                code=settings.RetCode.AUTHENTICATION_ERROR
+                code=RetCode.AUTHENTICATION_ERROR
             )
 
     tenants = UserTenantService.get_tenants_by_user_id(current_user.id)
@@ -318,7 +284,7 @@ def rm_tags(kb_id):
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
     e, kb = KnowledgebaseService.get_by_id(kb_id)
 
@@ -338,7 +304,7 @@ def rename_tags(kb_id):
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
     e, kb = KnowledgebaseService.get_by_id(kb_id)
 
@@ -356,7 +322,7 @@ def knowledge_graph(kb_id):
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
     _, kb = KnowledgebaseService.get_by_id(kb_id)
     req = {
@@ -396,7 +362,7 @@ def delete_knowledge_graph(kb_id):
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
     _, kb = KnowledgebaseService.get_by_id(kb_id)
     settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), kb_id)
@@ -413,7 +379,7 @@ def get_meta():
             return get_json_result(
                 data=False,
                 message='No authorization.',
-                code=settings.RetCode.AUTHENTICATION_ERROR
+                code=RetCode.AUTHENTICATION_ERROR
             )
     return get_json_result(data=DocumentService.get_meta_by_kbs(kb_ids))
 
@@ -426,7 +392,7 @@ def get_basic_info():
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
 
     basic_info = DocumentService.knowledgebase_basic_info(kb_id)
@@ -439,7 +405,7 @@ def get_basic_info():
 def list_pipeline_logs():
     kb_id = request.args.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
 
     keywords = request.args.get("keywords", "")
 
@@ -483,7 +449,7 @@ def list_pipeline_logs():
 def list_pipeline_dataset_logs():
     kb_id = request.args.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
 
     page_number = int(request.args.get("page", 0))
     items_per_page = int(request.args.get("page_size", 0))
@@ -517,7 +483,7 @@ def list_pipeline_dataset_logs():
 def delete_pipeline_logs():
     kb_id = request.args.get("kb_id")
     if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
 
     req = request.get_json()
     log_ids = req.get("log_ids", [])
@@ -532,7 +498,7 @@ def delete_pipeline_logs():
 def pipeline_log_detail():
     log_id = request.args.get("log_id")
     if not log_id:
-        return get_json_result(data=False, message='Lack of "Pipeline log ID"', code=settings.RetCode.ARGUMENT_ERROR)
+        return get_json_result(data=False, message='Lack of "Pipeline log ID"', code=RetCode.ARGUMENT_ERROR)
 
     ok, log = PipelineOperationLogService.get_by_id(log_id)
     if not ok:
@@ -762,29 +728,173 @@ def delete_kb_task():
     if not pipeline_task_type or pipeline_task_type not in [PipelineTaskType.GRAPH_RAG, PipelineTaskType.RAPTOR, PipelineTaskType.MINDMAP]:
         return get_error_data_result(message="Invalid task type")
 
+    def cancel_task(task_id):
+        REDIS_CONN.set(f"{task_id}-cancel", "x")
+
     match pipeline_task_type:
         case PipelineTaskType.GRAPH_RAG:
-            settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), kb_id)
             kb_task_id_field = "graphrag_task_id"
             task_id = kb.graphrag_task_id
             kb_task_finish_at = "graphrag_task_finish_at"
+            cancel_task(task_id)
+            settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), kb_id)
         case PipelineTaskType.RAPTOR:
             kb_task_id_field = "raptor_task_id"
             task_id = kb.raptor_task_id
             kb_task_finish_at = "raptor_task_finish_at"
+            cancel_task(task_id)
+            settings.docStoreConn.delete({"raptor_kwd": ["raptor"]}, search.index_name(kb.tenant_id), kb_id)
         case PipelineTaskType.MINDMAP:
             kb_task_id_field = "mindmap_task_id"
             task_id = kb.mindmap_task_id
             kb_task_finish_at = "mindmap_task_finish_at"
+            cancel_task(task_id)
         case _:
             return get_error_data_result(message="Internal Error: Invalid task type")
 
-    def cancel_task(task_id):
-        REDIS_CONN.set(f"{task_id}-cancel", "x")
-    cancel_task(task_id)
 
     ok = KnowledgebaseService.update_by_id(kb_id, {kb_task_id_field: "", kb_task_finish_at: None})
     if not ok:
         return server_error_response(f"Internal error: cannot delete task {pipeline_task_type}")
 
     return get_json_result(data=True)
+
+@manager.route("/check_embedding", methods=["post"])  # noqa: F821
+@login_required
+def check_embedding():
+
+    def _guess_vec_field(src: dict) -> str | None:
+        for k in src or {}:
+            if k.endswith("_vec"):
+                return k
+        return None
+
+    def _as_float_vec(v):
+        if v is None:
+            return []
+        if isinstance(v, str):
+            return [float(x) for x in v.split("\t") if x != ""]
+        if isinstance(v, (list, tuple, np.ndarray)):
+            return [float(x) for x in v]
+        return []
+
+    def _to_1d(x):
+        a = np.asarray(x, dtype=np.float32)
+        return a.reshape(-1)  
+
+    def _cos_sim(a, b, eps=1e-12):
+        a = _to_1d(a)
+        b = _to_1d(b)
+        na = np.linalg.norm(a)
+        nb = np.linalg.norm(b)
+        if na < eps or nb < eps: 
+            return 0.0
+        return float(np.dot(a, b) / (na * nb))
+
+    def sample_random_chunks_with_vectors(
+        docStoreConn,
+        tenant_id: str,
+        kb_id: str,
+        n: int = 5,
+        base_fields=("docnm_kwd","doc_id","content_with_weight","page_num_int","position_int","top_int"),
+    ):
+        index_nm = search.index_name(tenant_id)
+
+        res0 = docStoreConn.search(
+            selectFields=[], highlightFields=[],
+            condition={"kb_id": kb_id, "available_int": 1},
+            matchExprs=[], orderBy=OrderByExpr(),
+            offset=0, limit=1,
+            indexNames=index_nm, knowledgebaseIds=[kb_id]
+        )
+        total = docStoreConn.getTotal(res0)
+        if total <= 0:
+            return []
+
+        n = min(n, total)
+        offsets = sorted(random.sample(range(total), n))
+        out = []
+
+        for off in offsets:
+            res1 = docStoreConn.search(
+                selectFields=list(base_fields),
+                highlightFields=[],
+                condition={"kb_id": kb_id, "available_int": 1},
+                matchExprs=[], orderBy=OrderByExpr(),
+                offset=off, limit=1,
+                indexNames=index_nm, knowledgebaseIds=[kb_id]
+            )
+            ids = docStoreConn.getChunkIds(res1)
+            if not ids: 
+                continue
+
+            cid = ids[0]
+            full_doc = docStoreConn.get(cid, index_nm, [kb_id]) or {}
+            vec_field = _guess_vec_field(full_doc)
+            vec = _as_float_vec(full_doc.get(vec_field))
+
+            out.append({
+                "chunk_id": cid,
+                "kb_id": kb_id,
+                "doc_id": full_doc.get("doc_id"),
+                "doc_name": full_doc.get("docnm_kwd"),
+                "vector_field": vec_field,
+                "vector_dim": len(vec),
+                "vector": vec,
+                "page_num_int": full_doc.get("page_num_int"),
+                "position_int": full_doc.get("position_int"),
+                "top_int": full_doc.get("top_int"),
+                "content_with_weight": full_doc.get("content_with_weight") or "",
+            })
+        return out
+    req = request.json
+    kb_id = req.get("kb_id", "")
+    embd_id = req.get("embd_id", "")
+    n = int(req.get("check_num", 5))
+    _, kb = KnowledgebaseService.get_by_id(kb_id)
+    tenant_id = kb.tenant_id
+
+    emb_mdl = LLMBundle(tenant_id, LLMType.EMBEDDING, embd_id)
+    samples = sample_random_chunks_with_vectors(settings.docStoreConn, tenant_id=tenant_id, kb_id=kb_id, n=n)
+
+    results, eff_sims = [], []
+    for ck in samples:
+        txt = (ck.get("content_with_weight") or "").strip()
+        if not txt:
+            results.append({"chunk_id": ck["chunk_id"], "reason": "no_text"})
+            continue
+
+        if not ck.get("vector"):
+            results.append({"chunk_id": ck["chunk_id"], "reason": "no_stored_vector"})
+            continue
+
+        try:
+            qv, _ = emb_mdl.encode_queries(txt)  
+            sim = _cos_sim(qv, ck["vector"])
+        except Exception:
+            return get_error_data_result(message="embedding failure")
+
+        eff_sims.append(sim)
+        results.append({
+            "chunk_id": ck["chunk_id"],
+            "doc_id": ck["doc_id"],
+            "doc_name": ck["doc_name"],
+            "vector_field": ck["vector_field"],
+            "vector_dim": ck["vector_dim"],
+            "cos_sim": round(sim, 6),
+        })
+
+    summary = {
+        "kb_id": kb_id,
+        "model": embd_id,
+        "sampled": len(samples),
+        "valid": len(eff_sims),
+        "avg_cos_sim": round(float(np.mean(eff_sims)) if eff_sims else 0.0, 6),
+        "min_cos_sim": round(float(np.min(eff_sims)) if eff_sims else 0.0, 6),
+        "max_cos_sim": round(float(np.max(eff_sims)) if eff_sims else 0.0, 6),
+    }
+    if summary["avg_cos_sim"] > 0.99:
+        return get_json_result(data={"summary": summary, "results": results})
+    return get_json_result(code=RetCode.NOT_EFFECTIVE, message="failed", data={"summary": summary, "results": results})
+
+

api/apps/llm_app.py

@@ -15,24 +15,24 @@
 #
 import logging
 import json
+import os
 from flask import request
 from flask_login import login_required, current_user
 from api.db.services.tenant_llm_service import LLMFactoriesService, TenantLLMService
 from api.db.services.llm_service import LLMService
-from api import settings
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.db import StatusEnum, LLMType
+from common.constants import StatusEnum, LLMType
 from api.db.db_models import TenantLLM
-from api.utils.api_utils import get_json_result
-from api.utils.base64_image import test_image
+from api.utils.api_utils import get_json_result, get_allowed_llm_factories
+from rag.utils.base64_image import test_image
 from rag.llm import EmbeddingModel, ChatModel, RerankModel, CvModel, TTSModel
 
 
-@manager.route('/factories', methods=['GET'])  # noqa: F821
+@manager.route("/factories", methods=["GET"])  # noqa: F821
 @login_required
 def factories():
     try:
-        fac = LLMFactoriesService.get_all()
+        fac = get_allowed_llm_factories()
         fac = [f.to_dict() for f in fac if f.name not in ["Youdao", "FastEmbed", "BAAI"]]
         llms = LLMService.get_all()
         mdl_types = {}
@@ -43,14 +43,13 @@ def factories():
                 mdl_types[m.fid] = set([])
             mdl_types[m.fid].add(m.model_type)
         for f in fac:
-            f["model_types"] = list(mdl_types.get(f["name"], [LLMType.CHAT, LLMType.EMBEDDING, LLMType.RERANK,
-                                                              LLMType.IMAGE2TEXT, LLMType.SPEECH2TEXT, LLMType.TTS]))
+            f["model_types"] = list(mdl_types.get(f["name"], [LLMType.CHAT, LLMType.EMBEDDING, LLMType.RERANK, LLMType.IMAGE2TEXT, LLMType.SPEECH2TEXT, LLMType.TTS]))
         return get_json_result(data=fac)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/set_api_key', methods=['POST'])  # noqa: F821
+@manager.route("/set_api_key", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("llm_factory", "api_key")
 def set_api_key():
@@ -63,8 +62,7 @@ def set_api_key():
     for llm in LLMService.query(fid=factory):
         if not embd_passed and llm.model_type == LLMType.EMBEDDING.value:
             assert factory in EmbeddingModel, f"Embedding model from {factory} is not supported yet."
-            mdl = EmbeddingModel[factory](
-                req["api_key"], llm.llm_name, base_url=req.get("base_url"))
+            mdl = EmbeddingModel[factory](req["api_key"], llm.llm_name, base_url=req.get("base_url"))
             try:
                 arr, tc = mdl.encode(["Test if the api key is available"])
                 if len(arr[0]) == 0:
@@ -74,52 +72,40 @@ def set_api_key():
                 msg += f"\nFail to access embedding model({llm.llm_name}) using this api key." + str(e)
         elif not chat_passed and llm.model_type == LLMType.CHAT.value:
             assert factory in ChatModel, f"Chat model from {factory} is not supported yet."
-            mdl = ChatModel[factory](
-                req["api_key"], llm.llm_name, base_url=req.get("base_url"), **extra)
+            mdl = ChatModel[factory](req["api_key"], llm.llm_name, base_url=req.get("base_url"), **extra)
             try:
-                m, tc = mdl.chat(None, [{"role": "user", "content": "Hello! How are you doing!"}],
-                                 {"temperature": 0.9, 'max_tokens': 50})
+                m, tc = mdl.chat(None, [{"role": "user", "content": "Hello! How are you doing!"}], {"temperature": 0.9, "max_tokens": 50})
                 if m.find("**ERROR**") >= 0:
                     raise Exception(m)
                 chat_passed = True
             except Exception as e:
-                msg += f"\nFail to access model({llm.fid}/{llm.llm_name}) using this api key." + str(
-                    e)
+                msg += f"\nFail to access model({llm.fid}/{llm.llm_name}) using this api key." + str(e)
         elif not rerank_passed and llm.model_type == LLMType.RERANK:
             assert factory in RerankModel, f"Re-rank model from {factory} is not supported yet."
-            mdl = RerankModel[factory](
-                req["api_key"], llm.llm_name, base_url=req.get("base_url"))
+            mdl = RerankModel[factory](req["api_key"], llm.llm_name, base_url=req.get("base_url"))
             try:
                 arr, tc = mdl.similarity("What's the weather?", ["Is it sunny today?"])
                 if len(arr) == 0 or tc == 0:
                     raise Exception("Fail")
                 rerank_passed = True
-                logging.debug(f'passed model rerank {llm.llm_name}')
+                logging.debug(f"passed model rerank {llm.llm_name}")
             except Exception as e:
-                msg += f"\nFail to access model({llm.fid}/{llm.llm_name}) using this api key." + str(
-                    e)
+                msg += f"\nFail to access model({llm.fid}/{llm.llm_name}) using this api key." + str(e)
         if any([embd_passed, chat_passed, rerank_passed]):
-            msg = ''
+            msg = ""
             break
 
     if msg:
         return get_data_error_result(message=msg)
 
-    llm_config = {
-        "api_key": req["api_key"],
-        "api_base": req.get("base_url", "")
-    }
+    llm_config = {"api_key": req["api_key"], "api_base": req.get("base_url", "")}
     for n in ["model_type", "llm_name"]:
         if n in req:
             llm_config[n] = req[n]
 
     for llm in LLMService.query(fid=factory):
-        llm_config["max_tokens"]=llm.max_tokens
-        if not TenantLLMService.filter_update(
-                [TenantLLM.tenant_id == current_user.id,
-                 TenantLLM.llm_factory == factory,
-                 TenantLLM.llm_name == llm.llm_name],
-                llm_config):
+        llm_config["max_tokens"] = llm.max_tokens
+        if not TenantLLMService.filter_update([TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == factory, TenantLLM.llm_name == llm.llm_name], llm_config):
             TenantLLMService.save(
                 tenant_id=current_user.id,
                 llm_factory=factory,
@@ -127,13 +113,13 @@ def set_api_key():
                 model_type=llm.model_type,
                 api_key=llm_config["api_key"],
                 api_base=llm_config["api_base"],
-                max_tokens=llm_config["max_tokens"]
+                max_tokens=llm_config["max_tokens"],
             )
 
     return get_json_result(data=True)
 
 
-@manager.route('/add_llm', methods=['POST'])  # noqa: F821
+@manager.route("/add_llm", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("llm_factory")
 def add_llm():
@@ -142,6 +128,9 @@ def add_llm():
     api_key = req.get("api_key", "x")
     llm_name = req.get("llm_name")
 
+    if factory not in get_allowed_llm_factories():
+        return get_data_error_result(message=f"LLM factory {factory} is not allowed")
+
     def apikey_json(keys):
         nonlocal req
         return json.dumps({k: req.get(k, "") for k in keys})
@@ -204,7 +193,7 @@ def add_llm():
         "llm_name": llm_name,
         "api_base": req.get("api_base", ""),
         "api_key": api_key,
-        "max_tokens": req.get("max_tokens")
+        "max_tokens": req.get("max_tokens"),
     }
 
     msg = ""
@@ -212,10 +201,7 @@ def add_llm():
     extra = {"provider": factory}
     if llm["model_type"] == LLMType.EMBEDDING.value:
         assert factory in EmbeddingModel, f"Embedding model from {factory} is not supported yet."
-        mdl = EmbeddingModel[factory](
-            key=llm['api_key'],
-            model_name=mdl_nm,
-            base_url=llm["api_base"])
+        mdl = EmbeddingModel[factory](key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"])
         try:
             arr, tc = mdl.encode(["Test if the api key is available"])
             if len(arr[0]) == 0:
@@ -225,54 +211,41 @@ def add_llm():
     elif llm["model_type"] == LLMType.CHAT.value:
         assert factory in ChatModel, f"Chat model from {factory} is not supported yet."
         mdl = ChatModel[factory](
-            key=llm['api_key'],
+            key=llm["api_key"],
             model_name=mdl_nm,
             base_url=llm["api_base"],
             **extra,
         )
         try:
-            m, tc = mdl.chat(None, [{"role": "user", "content": "Hello! How are you doing!"}], {
-                "temperature": 0.9})
+            m, tc = mdl.chat(None, [{"role": "user", "content": "Hello! How are you doing!"}], {"temperature": 0.9})
             if not tc and m.find("**ERROR**:") >= 0:
                 raise Exception(m)
         except Exception as e:
-            msg += f"\nFail to access model({factory}/{mdl_nm})." + str(
-                e)
+            msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
     elif llm["model_type"] == LLMType.RERANK:
         assert factory in RerankModel, f"RE-rank model from {factory} is not supported yet."
         try:
-            mdl = RerankModel[factory](
-                key=llm["api_key"],
-                model_name=mdl_nm,
-                base_url=llm["api_base"]
-            )
+            mdl = RerankModel[factory](key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"])
             arr, tc = mdl.similarity("Hello~ RAGFlower!", ["Hi, there!", "Ohh, my friend!"])
             if len(arr) == 0:
                 raise Exception("Not known.")
         except KeyError:
             msg += f"{factory} dose not support this model({factory}/{mdl_nm})"
         except Exception as e:
-            msg += f"\nFail to access model({factory}/{mdl_nm})." + str(
-                e)
+            msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
     elif llm["model_type"] == LLMType.IMAGE2TEXT.value:
         assert factory in CvModel, f"Image to text model from {factory} is not supported yet."
-        mdl = CvModel[factory](
-            key=llm["api_key"],
-            model_name=mdl_nm,
-            base_url=llm["api_base"]
-        )
+        mdl = CvModel[factory](key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"])
         try:
             image_data = test_image
             m, tc = mdl.describe(image_data)
-            if not m and not tc:
+            if not tc and m.find("**ERROR**:") >= 0:
                 raise Exception(m)
         except Exception as e:
             msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
     elif llm["model_type"] == LLMType.TTS:
         assert factory in TTSModel, f"TTS model from {factory} is not supported yet."
-        mdl = TTSModel[factory](
-            key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"]
-        )
+        mdl = TTSModel[factory](key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"])
         try:
             for resp in mdl.tts("Hello~ RAGFlower!"):
                 pass
@@ -285,40 +258,46 @@ def add_llm():
     if msg:
         return get_data_error_result(message=msg)
 
-    if not TenantLLMService.filter_update(
-            [TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == factory,
-             TenantLLM.llm_name == llm["llm_name"]], llm):
+    if not TenantLLMService.filter_update([TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == factory, TenantLLM.llm_name == llm["llm_name"]], llm):
         TenantLLMService.save(**llm)
 
     return get_json_result(data=True)
 
 
-@manager.route('/delete_llm', methods=['POST'])  # noqa: F821
+@manager.route("/delete_llm", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("llm_factory", "llm_name")
 def delete_llm():
     req = request.json
-    TenantLLMService.filter_delete(
-        [TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"],
-         TenantLLM.llm_name == req["llm_name"]])
+    TenantLLMService.filter_delete([TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"], TenantLLM.llm_name == req["llm_name"]])
     return get_json_result(data=True)
 
 
-@manager.route('/delete_factory', methods=['POST'])  # noqa: F821
+@manager.route("/enable_llm", methods=["POST"])  # noqa: F821
+@login_required
+@validate_request("llm_factory", "llm_name")
+def enable_llm():
+    req = request.json
+    TenantLLMService.filter_update(
+        [TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"], TenantLLM.llm_name == req["llm_name"]], {"status": str(req.get("status", "1"))}
+    )
+    return get_json_result(data=True)
+
+
+@manager.route("/delete_factory", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("llm_factory")
 def delete_factory():
     req = request.json
-    TenantLLMService.filter_delete(
-        [TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"]])
+    TenantLLMService.filter_delete([TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"]])
     return get_json_result(data=True)
 
 
-@manager.route('/my_llms', methods=['GET'])  # noqa: F821
+@manager.route("/my_llms", methods=["GET"])  # noqa: F821
 @login_required
 def my_llms():
     try:
-        include_details = request.args.get('include_details', 'false').lower() == 'true'
+        include_details = request.args.get("include_details", "false").lower() == "true"
 
         if include_details:
             res = {}
@@ -334,51 +313,46 @@ def my_llms():
                         break
 
                 if o_dict["llm_factory"] not in res:
-                    res[o_dict["llm_factory"]] = {
-                        "tags": factory_tags,
-                        "llm": []
-                    }
+                    res[o_dict["llm_factory"]] = {"tags": factory_tags, "llm": []}
 
-                res[o_dict["llm_factory"]]["llm"].append({
-                    "type": o_dict["model_type"],
-                    "name": o_dict["llm_name"],
-                    "used_token": o_dict["used_tokens"],
-                    "api_base": o_dict["api_base"] or "",
-                    "max_tokens": o_dict["max_tokens"] or 8192
-                })
+                res[o_dict["llm_factory"]]["llm"].append(
+                    {
+                        "type": o_dict["model_type"],
+                        "name": o_dict["llm_name"],
+                        "used_token": o_dict["used_tokens"],
+                        "api_base": o_dict["api_base"] or "",
+                        "max_tokens": o_dict["max_tokens"] or 8192,
+                        "status": o_dict["status"] or "1",
+                    }
+                )
         else:
             res = {}
             for o in TenantLLMService.get_my_llms(current_user.id):
                 if o["llm_factory"] not in res:
-                    res[o["llm_factory"]] = {
-                        "tags": o["tags"],
-                        "llm": []
-                    }
-                res[o["llm_factory"]]["llm"].append({
-                    "type": o["model_type"],
-                    "name": o["llm_name"],
-                    "used_token": o["used_tokens"]
-                })
+                    res[o["llm_factory"]] = {"tags": o["tags"], "llm": []}
+                res[o["llm_factory"]]["llm"].append({"type": o["model_type"], "name": o["llm_name"], "used_token": o["used_tokens"], "status": o["status"]})
 
         return get_json_result(data=res)
     except Exception as e:
         return server_error_response(e)
 
 
-@manager.route('/list', methods=['GET'])  # noqa: F821
+@manager.route("/list", methods=["GET"])  # noqa: F821
 @login_required
 def list_app():
-    self_deployed = ["Youdao", "FastEmbed", "BAAI", "Ollama", "Xinference", "LocalAI", "LM-Studio", "GPUStack"]
-    weighted = ["Youdao", "FastEmbed", "BAAI"] if settings.LIGHTEN != 0 else []
+    self_deployed = ["FastEmbed", "Ollama", "Xinference", "LocalAI", "LM-Studio", "GPUStack"]
+    weighted = []
     model_type = request.args.get("model_type")
     try:
         objs = TenantLLMService.query(tenant_id=current_user.id)
-        facts = set([o.to_dict()["llm_factory"] for o in objs if o.api_key])
+        facts = set([o.to_dict()["llm_factory"] for o in objs if o.api_key and o.status == StatusEnum.VALID.value])
+        status = {(o.llm_name + "@" + o.llm_factory) for o in objs if o.status == StatusEnum.VALID.value}
         llms = LLMService.get_all()
-        llms = [m.to_dict()
-                for m in llms if m.status == StatusEnum.VALID.value and m.fid not in weighted]
+        llms = [m.to_dict() for m in llms if m.status == StatusEnum.VALID.value and m.fid not in weighted and (m.llm_name + "@" + m.fid) in status]
         for m in llms:
             m["available"] = m["fid"] in facts or m["llm_name"].lower() == "flag-embedding" or m["fid"] in self_deployed
+            if "tei-" in os.getenv("COMPOSE_PROFILES", "") and m["model_type"] == LLMType.EMBEDDING and m["fid"] == "Builtin" and m["llm_name"] == os.getenv("TEI_MODEL", ""):
+                m["available"] = True
 
         llm_set = set([m["llm_name"] + "@" + m["fid"] for m in llms])
         for o in objs:

api/apps/mcp_server_app.py

@@ -16,13 +16,12 @@
 from flask import Response, request
 from flask_login import current_user, login_required
 
-from api.db import VALID_MCP_SERVER_TYPES
 from api.db.db_models import MCPServer
 from api.db.services.mcp_server_service import MCPServerService
 from api.db.services.user_service import TenantService
-from api.settings import RetCode
+from common.constants import RetCode, VALID_MCP_SERVER_TYPES
 
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request, \
     get_mcp_tools
 from api.utils.web_utils import get_float, safe_json_parse

api/apps/sdk/agent.py

@@ -15,15 +15,19 @@
 #
 
 import json
+import logging
 import time
 from typing import Any, cast
+
+from agent.canvas import Canvas
+from api.db import CanvasCategory
 from api.db.services.canvas_service import UserCanvasService
 from api.db.services.user_canvas_version import UserCanvasVersionService
-from api.settings import RetCode
-from api.utils import get_uuid
+from common.constants import RetCode
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_data_error_result, get_error_data_result, get_json_result, token_required
 from api.utils.api_utils import get_result
-from flask import request
+from flask import request, Response
 
 
 @manager.route('/agents', methods=['GET'])  # noqa: F821
@@ -127,3 +131,49 @@ def delete_agent(tenant_id: str, agent_id: str):
 
     UserCanvasService.delete_by_id(agent_id)
     return get_json_result(data=True)
+
+
+@manager.route('/webhook/<agent_id>', methods=['POST'])  # noqa: F821
+@token_required
+def webhook(tenant_id: str, agent_id: str):
+    req = request.json
+    if not UserCanvasService.accessible(req["id"], tenant_id):
+        return get_json_result(
+            data=False, message='Only owner of canvas authorized for this operation.',
+            code=RetCode.OPERATING_ERROR)
+
+    e, cvs = UserCanvasService.get_by_id(req["id"])
+    if not e:
+        return get_data_error_result(message="canvas not found.")
+
+    if not isinstance(cvs.dsl, str):
+        cvs.dsl = json.dumps(cvs.dsl, ensure_ascii=False)
+
+    if cvs.canvas_category == CanvasCategory.DataFlow:
+        return get_data_error_result(message="Dataflow can not be triggered by webhook.")
+
+    try:
+        canvas = Canvas(cvs.dsl, tenant_id, agent_id)
+    except Exception as e:
+        return get_json_result(
+            data=False, message=str(e),
+            code=RetCode.EXCEPTION_ERROR)
+
+    def sse():
+        nonlocal canvas
+        try:
+            for ans in canvas.run(query=req.get("query", ""), files=req.get("files", []), user_id=req.get("user_id", tenant_id), webhook_payload=req):
+                yield "data:" + json.dumps(ans, ensure_ascii=False) + "\n\n"
+
+            cvs.dsl = json.loads(str(canvas))
+            UserCanvasService.update_by_id(req["id"], cvs.to_dict())
+        except Exception as e:
+            logging.exception(e)
+            yield "data:" + json.dumps({"code": 500, "message": str(e), "data": False}, ensure_ascii=False) + "\n\n"
+
+    resp = Response(sse(), mimetype="text/event-stream")
+    resp.headers.add_header("Cache-control", "no-cache")
+    resp.headers.add_header("Connection", "keep-alive")
+    resp.headers.add_header("X-Accel-Buffering", "no")
+    resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
+    return resp

api/apps/sdk/chat.py

@@ -17,13 +17,12 @@ import logging
 
 from flask import request
 
-from api import settings
-from api.db import StatusEnum
 from api.db.services.dialog_service import DialogService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_service import TenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
+from common.constants import RetCode, StatusEnum
 from api.utils.api_utils import check_duplicate_ids, get_error_data_result, get_result, token_required
 
 
@@ -45,7 +44,7 @@ def create(tenant_id):
     embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]  # remove vendor suffix for comparison
     embd_count = list(set(embd_ids))
     if len(embd_count) > 1:
-        return get_result(message='Datasets use different embedding models."', code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_result(message='Datasets use different embedding models."', code=RetCode.AUTHENTICATION_ERROR)
     req["kb_ids"] = ids
     # llm
     llm = req.get("llm")
@@ -167,8 +166,10 @@ def update(tenant_id, chat_id):
         embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]  # remove vendor suffix for comparison
         embd_count = list(set(embd_ids))
         if len(embd_count) > 1:
-            return get_result(message='Datasets use different embedding models."', code=settings.RetCode.AUTHENTICATION_ERROR)
+            return get_result(message='Datasets use different embedding models."', code=RetCode.AUTHENTICATION_ERROR)
         req["kb_ids"] = ids
+    else:
+        req["kb_ids"] = []
     llm = req.get("llm")
     if llm:
         if "model_name" in llm:

api/apps/sdk/dataset.py

@@ -20,20 +20,17 @@ import os
 import json
 from flask import request
 from peewee import OperationalError
-from api import settings
-from api.db import FileSource, StatusEnum
 from api.db.db_models import File
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import TenantService
-from api.utils import get_uuid
+from common.constants import RetCode, FileSource, StatusEnum
 from api.utils.api_utils import (
     deep_merge,
     get_error_argument_result,
     get_error_data_result,
-    get_error_operating_result,
     get_error_permission_result,
     get_parser_config,
     get_result,
@@ -50,7 +47,8 @@ from api.utils.validation_utils import (
     validate_and_parse_request_args,
 )
 from rag.nlp import search
-from rag.settings import PAGERANK_FLD
+from common.constants import PAGERANK_FLD
+from common import settings
 
 
 @manager.route("/datasets", methods=["POST"])  # noqa: F821
@@ -80,29 +78,28 @@ def create(tenant_id):
           properties:
             name:
               type: string
-              description: Name of the dataset.
+              description: Dataset name (required).
             avatar:
               type: string
-              description: Base64 encoding of the avatar.
+              description: Optional base64-encoded avatar image.
             description:
               type: string
-              description: Description of the dataset.
+              description: Optional dataset description.
             embedding_model:
               type: string
-              description: Embedding model Name.
+              description: Optional embedding model name; if omitted, the tenant's default embedding model is used.
             permission:
               type: string
               enum: ['me', 'team']
-              description: Dataset permission.
+              description: Visibility of the dataset (private to me or shared with team).
             chunk_method:
               type: string
               enum: ["naive", "book", "email", "laws", "manual", "one", "paper",
-                     "picture", "presentation", "qa", "table", "tag"
-                     ]
-              description: Chunking method.
+                     "picture", "presentation", "qa", "table", "tag"]
+              description: Chunking method; if omitted, defaults to "naive".
             parser_config:
               type: object
-              description: Parser configuration.
+              description: Optional parser configuration; server-side defaults will be applied.
     responses:
       200:
         description: Successful operation.
@@ -117,44 +114,43 @@ def create(tenant_id):
     # |----------------|-------------|
     # | embedding_model| embd_id     |
     # | chunk_method   | parser_id   |
+
     req, err = validate_and_parse_json_request(request, CreateDatasetReq)
     if err is not None:
         return get_error_argument_result(err)
+    
+    req = KnowledgebaseService.create_with_name(
+        name = req.pop("name", None),
+        tenant_id = tenant_id,
+        parser_id = req.pop("parser_id", None),
+        **req
+    )
+
+    # Insert embedding model(embd id)
+    ok, t = TenantService.get_by_id(tenant_id)
+    if not ok:
+        return get_error_permission_result(message="Tenant not found")
+    if not req.get("embd_id"):
+        req["embd_id"] = t.embd_id
+    else:
+        ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
+        if not ok:
+            return err
+
 
     try:
-        if KnowledgebaseService.get_or_none(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
-            return get_error_operating_result(message=f"Dataset name '{req['name']}' already exists")
-
-        req["parser_config"] = get_parser_config(req["parser_id"], req["parser_config"])
-        req["id"] = get_uuid()
-        req["tenant_id"] = tenant_id
-        req["created_by"] = tenant_id
-
-        ok, t = TenantService.get_by_id(tenant_id)
-        if not ok:
-            return get_error_permission_result(message="Tenant not found")
-
-        if not req.get("embd_id"):
-            req["embd_id"] = t.embd_id
-        else:
-            ok, err = verify_embedding_availability(req["embd_id"], tenant_id)
-            if not ok:
-                return err
-
-        if not KnowledgebaseService.save(**req):
-            return get_error_data_result(message="Create dataset error.(Database error)")
-
-        ok, k = KnowledgebaseService.get_by_id(req["id"])
-        if not ok:
-            return get_error_data_result(message="Dataset created failed")
-
-        response_data = remap_dictionary_keys(k.to_dict())
-        return get_result(data=response_data)
-    except OperationalError as e:
+      if not KnowledgebaseService.save(**req):
+          return get_error_data_result()
+      ok, k = KnowledgebaseService.get_by_id(req["id"])
+      if not ok:
+        return get_error_data_result(message="Dataset created failed")
+    
+      response_data = remap_dictionary_keys(k.to_dict())
+      return get_result(data=response_data)
+    except Exception as e:
         logging.exception(e)
         return get_error_data_result(message="Database operation failed")
 
-
 @manager.route("/datasets", methods=["DELETE"])  # noqa: F821
 @token_required
 def delete(tenant_id):
@@ -488,7 +484,7 @@ def knowledge_graph(tenant_id, dataset_id):
         return get_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
     _, kb = KnowledgebaseService.get_by_id(dataset_id)
     req = {
@@ -529,7 +525,7 @@ def delete_knowledge_graph(tenant_id, dataset_id):
         return get_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR
+            code=RetCode.AUTHENTICATION_ERROR
         )
     _, kb = KnowledgebaseService.get_by_id(dataset_id)
     settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]},

api/apps/sdk/dify_retrieval.py

@@ -17,15 +17,14 @@ import logging
 
 from flask import request, jsonify
 
-from api.db import LLMType
 from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
-from api import settings
 from api.utils.api_utils import validate_request, build_error_result, apikey_required
 from rag.app.tag import label_question
 from api.db.services.dialog_service import meta_filter, convert_conditions
-
+from common.constants import RetCode, LLMType
+from common import settings
 
 @manager.route('/dify/retrieval', methods=['POST'])  # noqa: F821
 @apikey_required
@@ -129,7 +128,7 @@ def retrieval(tenant_id):
 
         e, kb = KnowledgebaseService.get_by_id(kb_id)
         if not e:
-            return build_error_result(message="Knowledgebase not found!", code=settings.RetCode.NOT_FOUND)
+            return build_error_result(message="Knowledgebase not found!", code=RetCode.NOT_FOUND)
 
         embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
         print(metadata_condition)
@@ -179,7 +178,7 @@ def retrieval(tenant_id):
         if str(e).find("not_found") > 0:
             return build_error_result(
                 message='No chunk found! Check the chunk status please!',
-                code=settings.RetCode.NOT_FOUND
+                code=RetCode.NOT_FOUND
             )
         logging.exception(e)
-        return build_error_result(message=str(e), code=settings.RetCode.SERVER_ERROR)
+        return build_error_result(message=str(e), code=RetCode.SERVER_ERROR)

api/apps/sdk/doc.py

@@ -24,9 +24,8 @@ from flask import request, send_file
 from peewee import OperationalError
 from pydantic import BaseModel, Field, validator
 
-from api import settings
 from api.constants import FILE_NAME_LEN_LIMIT
-from api.db import FileSource, FileType, LLMType, ParserType, TaskStatus
+from api.db import FileType
 from api.db.db_models import File, Task
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
@@ -41,8 +40,9 @@ from rag.app.qa import beAdoc, rmPrefix
 from rag.app.tag import label_question
 from rag.nlp import rag_tokenizer, search
 from rag.prompts.generator import cross_languages, keyword_extraction
-from rag.utils import rmSpace
-from rag.utils.storage_factory import STORAGE_IMPL
+from common.string_utils import remove_redundant_spaces
+from common.constants import RetCode, LLMType, ParserType, TaskStatus, FileSource
+from common import settings
 
 MAXIMUM_OF_UPLOADING_FILES = 256
 
@@ -127,13 +127,13 @@ def upload(dataset_id, tenant_id):
                     description: Processing status.
     """
     if "file" not in request.files:
-        return get_error_data_result(message="No file part!", code=settings.RetCode.ARGUMENT_ERROR)
+        return get_error_data_result(message="No file part!", code=RetCode.ARGUMENT_ERROR)
     file_objs = request.files.getlist("file")
     for file_obj in file_objs:
         if file_obj.filename == "":
-            return get_result(message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_result(message="No file selected!", code=RetCode.ARGUMENT_ERROR)
         if len(file_obj.filename.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
-            return get_result(message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=settings.RetCode.ARGUMENT_ERROR)
+            return get_result(message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
     """
     # total size
     total_size = 0
@@ -145,7 +145,7 @@ def upload(dataset_id, tenant_id):
     if total_size > MAX_TOTAL_FILE_SIZE:
         return get_result(
             message=f"Total file size exceeds 10MB limit! ({total_size / (1024 * 1024):.2f} MB)",
-            code=settings.RetCode.ARGUMENT_ERROR,
+            code=RetCode.ARGUMENT_ERROR,
         )
     """
     e, kb = KnowledgebaseService.get_by_id(dataset_id)
@@ -153,7 +153,7 @@ def upload(dataset_id, tenant_id):
         raise LookupError(f"Can't find the dataset with ID {dataset_id}!")
     err, files = FileService.upload_document(kb, file_objs, tenant_id)
     if err:
-        return get_result(message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
+        return get_result(message="\n".join(err), code=RetCode.SERVER_ERROR)
     # rename key's name
     renamed_doc_list = []
     for file in files:
@@ -253,12 +253,12 @@ def update_doc(tenant_id, dataset_id, document_id):
         if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
             return get_result(
                 message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.",
-                code=settings.RetCode.ARGUMENT_ERROR,
+                code=RetCode.ARGUMENT_ERROR,
             )
         if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
             return get_result(
                 message="The extension of file can't be changed",
-                code=settings.RetCode.ARGUMENT_ERROR,
+                code=RetCode.ARGUMENT_ERROR,
             )
         for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
             if d.name == req["name"]:
@@ -400,9 +400,9 @@ def download(tenant_id, dataset_id, document_id):
         return get_error_data_result(message=f"The dataset not own the document {document_id}.")
     # The process of downloading
     doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id)  # minio address
-    file_stream = STORAGE_IMPL.get(doc_id, doc_location)
+    file_stream = settings.STORAGE_IMPL.get(doc_id, doc_location)
     if not file_stream:
-        return construct_json_result(message="This file is empty.", code=settings.RetCode.DATA_ERROR)
+        return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR)
     file = BytesIO(file_stream)
     # Use send_file with a proper filename and MIME type
     return send_file(
@@ -670,16 +670,16 @@ def delete(tenant_id, dataset_id):
             )
             File2DocumentService.delete_by_document_id(doc_id)
 
-            STORAGE_IMPL.rm(b, n)
+            settings.STORAGE_IMPL.rm(b, n)
             success_count += 1
         except Exception as e:
             errors += str(e)
 
     if not_found:
-        return get_result(message=f"Documents not found: {not_found}", code=settings.RetCode.DATA_ERROR)
+        return get_result(message=f"Documents not found: {not_found}", code=RetCode.DATA_ERROR)
 
     if errors:
-        return get_result(message=errors, code=settings.RetCode.SERVER_ERROR)
+        return get_result(message=errors, code=RetCode.SERVER_ERROR)
 
     if duplicate_messages:
         if success_count > 0:
@@ -763,7 +763,7 @@ def parse(tenant_id, dataset_id):
         queue_tasks(doc, bucket, name, 0)
         success_count += 1
     if not_found:
-        return get_result(message=f"Documents not found: {not_found}", code=settings.RetCode.DATA_ERROR)
+        return get_result(message=f"Documents not found: {not_found}", code=RetCode.DATA_ERROR)
     if duplicate_messages:
         if success_count > 0:
             return get_result(
@@ -969,7 +969,7 @@ def list_chunks(tenant_id, dataset_id, document_id):
     if req.get("id"):
         chunk = settings.docStoreConn.get(req.get("id"), search.index_name(tenant_id), [dataset_id])
         if not chunk:
-            return get_result(message=f"Chunk not found: {dataset_id}/{req.get('id')}", code=settings.RetCode.NOT_FOUND)
+            return get_result(message=f"Chunk not found: {dataset_id}/{req.get('id')}", code=RetCode.NOT_FOUND)
         k = []
         for n in chunk.keys():
             if re.search(r"(_vec$|_sm_|_tks|_ltks)", n):
@@ -1000,7 +1000,7 @@ def list_chunks(tenant_id, dataset_id, document_id):
         for id in sres.ids:
             d = {
                 "id": id,
-                "content": (rmSpace(sres.highlight[id]) if question and id in sres.highlight else sres.field[id].get("content_with_weight", "")),
+                "content": (remove_redundant_spaces(sres.highlight[id]) if question and id in sres.highlight else sres.field[id].get("content_with_weight", "")),
                 "document_id": sres.field[id]["doc_id"],
                 "docnm_kwd": sres.field[id]["docnm_kwd"],
                 "important_keywords": sres.field[id].get("important_kwd", []),
@@ -1301,6 +1301,10 @@ def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
         d["question_tks"] = rag_tokenizer.tokenize("\n".join(req["questions"]))
     if "available" in req:
         d["available_int"] = int(req["available"])
+    if "positions" in req:
+        if not isinstance(req["positions"], list):
+            return get_error_data_result("`positions` should be a list")
+        d["position_int"] = req["positions"]
     embd_id = DocumentService.get_embd_id(document_id)
     embd_mdl = TenantLLMService.model_instance(tenant_id, LLMType.EMBEDDING.value, embd_id)
     if doc.parser_id == ParserType.QA:
@@ -1414,7 +1418,7 @@ def retrieval_test(tenant_id):
     if len(embd_nms) != 1:
         return get_result(
             message='Datasets use different embedding models."',
-            code=settings.RetCode.DATA_ERROR,
+            code=RetCode.DATA_ERROR,
         )
     if "question" not in req:
         return get_error_data_result("`question` is required.")
@@ -1505,6 +1509,6 @@ def retrieval_test(tenant_id):
         if str(e).find("not_found") > 0:
             return get_result(
                 message="No chunk found! Check the chunk status please!",
-                code=settings.RetCode.DATA_ERROR,
+                code=RetCode.DATA_ERROR,
             )
         return server_error_response(e)

api/apps/sdk/files.py

@@ -26,13 +26,13 @@ from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.utils.api_utils import server_error_response, token_required
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.db import FileType
 from api.db.services import duplicate_name
 from api.db.services.file_service import FileService
 from api.utils.api_utils import get_json_result
 from api.utils.file_utils import filename_type
-from rag.utils.storage_factory import STORAGE_IMPL
+from common import settings
 
 
 @manager.route('/file/upload', methods=['POST'])  # noqa: F821
@@ -126,7 +126,7 @@ def upload(tenant_id):
 
             filetype = filename_type(file_obj_names[file_len - 1])
             location = file_obj_names[file_len - 1]
-            while STORAGE_IMPL.obj_exist(last_folder.id, location):
+            while settings.STORAGE_IMPL.obj_exist(last_folder.id, location):
                 location += "_"
             blob = file_obj.read()
             filename = duplicate_name(FileService.query, name=file_obj_names[file_len - 1], parent_id=last_folder.id)
@@ -142,7 +142,7 @@ def upload(tenant_id):
                 "size": len(blob),
             }
             file = FileService.insert(file)
-            STORAGE_IMPL.put(last_folder.id, location, blob)
+            settings.STORAGE_IMPL.put(last_folder.id, location, blob)
             file_res.append(file.to_json())
         return get_json_result(data=file_res)
     except Exception as e:
@@ -497,10 +497,10 @@ def rm(tenant_id):
                     e, file = FileService.get_by_id(inner_file_id)
                     if not e:
                         return get_json_result(message="File not found!", code=404)
-                    STORAGE_IMPL.rm(file.parent_id, file.location)
+                    settings.STORAGE_IMPL.rm(file.parent_id, file.location)
                 FileService.delete_folder_by_pf_id(tenant_id, file_id)
             else:
-                STORAGE_IMPL.rm(file.parent_id, file.location)
+                settings.STORAGE_IMPL.rm(file.parent_id, file.location)
                 if not FileService.delete(file):
                     return get_json_result(message="Database error (File removal)!", code=500)
 
@@ -614,10 +614,10 @@ def get(tenant_id, file_id):
         if not e:
             return get_json_result(message="Document not found!", code=404)
 
-        blob = STORAGE_IMPL.get(file.parent_id, file.location)
+        blob = settings.STORAGE_IMPL.get(file.parent_id, file.location)
         if not blob:
             b, n = File2DocumentService.get_storage_address(file_id=file_id)
-            blob = STORAGE_IMPL.get(b, n)
+            blob = settings.STORAGE_IMPL.get(b, n)
 
         response = flask.make_response(blob)
         ext = re.search(r"\.([^.]+)$", file.name)

api/apps/sdk/session.py

@@ -21,11 +21,9 @@ import tiktoken
 from flask import Response, jsonify, request
 
 from agent.canvas import Canvas
-from api import settings
-from api.db import LLMType, StatusEnum
 from api.db.db_models import APIToken
 from api.db.services.api_service import API4ConversationService
-from api.db.services.canvas_service import UserCanvasService, completionOpenAI
+from api.db.services.canvas_service import UserCanvasService, completion_openai
 from api.db.services.canvas_service import completion as agent_completion
 from api.db.services.conversation_service import ConversationService, iframe_completion
 from api.db.services.conversation_service import completion as rag_completion
@@ -35,13 +33,14 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api.db.services.search_service import SearchService
 from api.db.services.user_service import UserTenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import check_duplicate_ids, get_data_openai, get_error_data_result, get_json_result, \
     get_result, server_error_response, token_required, validate_request
 from rag.app.tag import label_question
 from rag.prompts.template import load_prompt
 from rag.prompts.generator import cross_languages, gen_meta_filter, keyword_extraction, chunks_format
-
+from common.constants import RetCode, LLMType, StatusEnum
+from common import settings
 
 @manager.route("/chats/<chat_id>/sessions", methods=["POST"])  # noqa: F821
 @token_required
@@ -412,7 +411,7 @@ def agents_completion_openai_compatibility(tenant_id, agent_id):
     stream = req.pop("stream", False)
     if stream:
         resp = Response(
-            completionOpenAI(
+            completion_openai(
                 tenant_id,
                 agent_id,
                 question,
@@ -430,7 +429,7 @@ def agents_completion_openai_compatibility(tenant_id, agent_id):
     else:
         # For non-streaming, just return the response directly
         response = next(
-            completionOpenAI(
+            completion_openai(
                 tenant_id,
                 agent_id,
                 question,
@@ -959,7 +958,7 @@ def retrieval_test_embedded():
         kb_ids = [kb_ids]
     if not kb_ids:
         return get_json_result(data=False, message='Please specify dataset firstly.',
-                               code=settings.RetCode.DATA_ERROR)
+                               code=RetCode.DATA_ERROR)
     doc_ids = req.get("doc_ids", [])
     similarity_threshold = float(req.get("similarity_threshold", 0.0))
     vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
@@ -996,7 +995,7 @@ def retrieval_test_embedded():
                     break
             else:
                 return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.",
-                                       code=settings.RetCode.OPERATING_ERROR)
+                                       code=RetCode.OPERATING_ERROR)
 
         e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
         if not e:
@@ -1034,7 +1033,7 @@ def retrieval_test_embedded():
     except Exception as e:
         if str(e).find("not_found") > 0:
             return get_json_result(data=False, message="No chunk found! Check the chunk status please!",
-                                   code=settings.RetCode.DATA_ERROR)
+                                   code=RetCode.DATA_ERROR)
         return server_error_response(e)
 
 
@@ -1104,7 +1103,7 @@ def detail_share_embedded():
                 break
         else:
             return get_json_result(data=False, message="Has no permission for this operation.",
-                                   code=settings.RetCode.OPERATING_ERROR)
+                                   code=RetCode.OPERATING_ERROR)
 
         search = SearchService.get_detail(search_id)
         if not search:

api/apps/search_app.py

@@ -17,14 +17,13 @@
 from flask import request
 from flask_login import current_user, login_required
 
-from api import settings
 from api.constants import DATASET_NAME_LIMIT
-from api.db import StatusEnum
 from api.db.db_models import DB
 from api.db.services import duplicate_name
 from api.db.services.search_service import SearchService
 from api.db.services.user_service import TenantService, UserTenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
+from common.constants import RetCode, StatusEnum
 from api.utils.api_utils import get_data_error_result, get_json_result, not_allowed_parameters, server_error_response, validate_request
 
 
@@ -82,12 +81,12 @@ def update():
 
     search_id = req["search_id"]
     if not SearchService.accessible4deletion(search_id, current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
     try:
         search_app = SearchService.query(tenant_id=tenant_id, id=search_id)[0]
         if not search_app:
-            return get_json_result(data=False, message=f"Cannot find search {search_id}", code=settings.RetCode.DATA_ERROR)
+            return get_json_result(data=False, message=f"Cannot find search {search_id}", code=RetCode.DATA_ERROR)
 
         if req["name"].lower() != search_app.name.lower() and len(SearchService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) >= 1:
             return get_data_error_result(message="Duplicated search name.")
@@ -129,7 +128,7 @@ def detail():
             if SearchService.query(tenant_id=tenant.tenant_id, id=search_id):
                 break
         else:
-            return get_json_result(data=False, message="Has no permission for this operation.", code=settings.RetCode.OPERATING_ERROR)
+            return get_json_result(data=False, message="Has no permission for this operation.", code=RetCode.OPERATING_ERROR)
 
         search = SearchService.get_detail(search_id)
         if not search:
@@ -178,7 +177,7 @@ def rm():
     req = request.get_json()
     search_id = req["search_id"]
     if not SearchService.accessible4deletion(search_id, current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=settings.RetCode.AUTHENTICATION_ERROR)
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
     try:
         if not SearchService.delete_by_id(search_id):

api/apps/system_app.py

@@ -23,8 +23,6 @@ from api.db.db_models import APIToken
 from api.db.services.api_service import APITokenService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import UserTenantService
-from api import settings
-from api.utils import current_timestamp, datetime_format
 from api.utils.api_utils import (
     get_json_result,
     get_data_error_result,
@@ -32,12 +30,13 @@ from api.utils.api_utils import (
     generate_confirmation_token,
 )
 from api.versions import get_ragflow_version
-from rag.utils.storage_factory import STORAGE_IMPL, STORAGE_IMPL_TYPE
+from common.time_utils import current_timestamp, datetime_format
 from timeit import default_timer as timer
 
 from rag.utils.redis_conn import REDIS_CONN
 from flask import jsonify
 from api.utils.health_utils import run_health_checks
+from common import settings
 
 
 @manager.route("/version", methods=["GET"])  # noqa: F821
@@ -112,15 +111,15 @@ def status():
 
     st = timer()
     try:
-        STORAGE_IMPL.health()
+        settings.STORAGE_IMPL.health()
         res["storage"] = {
-            "storage": STORAGE_IMPL_TYPE.lower(),
+            "storage": settings.STORAGE_IMPL_TYPE.lower(),
             "status": "green",
             "elapsed": "{:.1f}".format((timer() - st) * 1000.0),
         }
     except Exception as e:
         res["storage"] = {
-            "storage": STORAGE_IMPL_TYPE.lower(),
+            "storage": settings.STORAGE_IMPL_TYPE.lower(),
             "status": "red",
             "elapsed": "{:.1f}".format((timer() - st) * 1000.0),
             "error": str(e),
@@ -217,8 +216,8 @@ def new_token():
         tenant_id = [tenant for tenant in tenants if tenant.role == 'owner'][0].tenant_id
         obj = {
             "tenant_id": tenant_id,
-            "token": generate_confirmation_token(tenant_id),
-            "beta": generate_confirmation_token(generate_confirmation_token(tenant_id)).replace("ragflow-", "")[:32],
+            "token": generate_confirmation_token(),
+            "beta": generate_confirmation_token().replace("ragflow-", "")[:32],
             "create_time": current_timestamp(),
             "create_date": datetime_format(datetime.now()),
             "update_time": None,
@@ -274,7 +273,7 @@ def token_list():
         objs = [o.to_dict() for o in objs]
         for o in objs:
             if not o["beta"]:
-                o["beta"] = generate_confirmation_token(generate_confirmation_token(tenants[0].tenant_id)).replace(
+                o["beta"] = generate_confirmation_token().replace(
                     "ragflow-", "")[:32]
                 APITokenService.filter_update([APIToken.tenant_id == tenant_id, APIToken.token == o["token"]], o)
         return get_json_result(data=objs)

api/apps/tenant_app.py

@@ -17,15 +17,17 @@
 from flask import request
 from flask_login import login_required, current_user
 
-from api import settings
 from api.apps import smtp_mail_server
-from api.db import UserTenantRole, StatusEnum
+from api.db import UserTenantRole
 from api.db.db_models import UserTenant
 from api.db.services.user_service import UserTenantService, UserService
 
-from api.utils import get_uuid, delta_seconds
+from common.constants import RetCode, StatusEnum
+from common.misc_utils import get_uuid
+from common.time_utils import delta_seconds
 from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result
 from api.utils.web_utils import send_invite_email
+from common import settings
 
 
 @manager.route("/<tenant_id>/user/list", methods=["GET"])  # noqa: F821
@@ -35,7 +37,7 @@ def user_list(tenant_id):
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR)
+            code=RetCode.AUTHENTICATION_ERROR)
 
     try:
         users = UserTenantService.get_by_tenant_id(tenant_id)
@@ -54,7 +56,7 @@ def create(tenant_id):
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR)
+            code=RetCode.AUTHENTICATION_ERROR)
 
     req = request.json
     invite_user_email = req["email"]
@@ -108,7 +110,7 @@ def rm(tenant_id, user_id):
         return get_json_result(
             data=False,
             message='No authorization.',
-            code=settings.RetCode.AUTHENTICATION_ERROR)
+            code=RetCode.AUTHENTICATION_ERROR)
 
     try:
         UserTenantService.filter_delete([UserTenant.tenant_id == tenant_id, UserTenant.user_id == user_id])

api/apps/user_app.py

@@ -26,7 +26,6 @@ from flask import redirect, request, session, make_response
 from flask_login import current_user, login_required, login_user, logout_user
 from werkzeug.security import check_password_hash, generate_password_hash
 
-from api import settings
 from api.apps.auth import get_auth_client
 from api.db import FileType, UserTenantRole
 from api.db.db_models import TenantLLM
@@ -34,15 +33,11 @@ from api.db.services.file_service import FileService
 from api.db.services.llm_service import get_init_tenant_llm
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_service import TenantService, UserService, UserTenantService
-from api.utils import (
-    current_timestamp,
-    datetime_format,
-    download_img,
-    get_format_time,
-    get_uuid,
-)
+from common.time_utils import current_timestamp, datetime_format, get_format_time
+from common.misc_utils import download_img, get_uuid
+from common.constants import RetCode
+from common.connection_utils import construct_response
 from api.utils.api_utils import (
-    construct_response,
     get_data_error_result,
     get_json_result,
     server_error_response,
@@ -62,6 +57,7 @@ from api.utils.web_utils import (
     hash_code,
     captcha_key,
 )
+from common import settings
 
 
 @manager.route("/login", methods=["POST", "GET"])  # noqa: F821
@@ -96,14 +92,14 @@ def login():
           type: object
     """
     if not request.json:
-        return get_json_result(data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="Unauthorized!")
+        return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="Unauthorized!")
 
     email = request.json.get("email", "")
     users = UserService.query(email=email)
     if not users:
         return get_json_result(
             data=False,
-            code=settings.RetCode.AUTHENTICATION_ERROR,
+            code=RetCode.AUTHENTICATION_ERROR,
             message=f"Email: {email} is not registered!",
         )
 
@@ -111,14 +107,14 @@ def login():
     try:
         password = decrypt(password)
     except BaseException:
-        return get_json_result(data=False, code=settings.RetCode.SERVER_ERROR, message="Fail to crypt password")
+        return get_json_result(data=False, code=RetCode.SERVER_ERROR, message="Fail to crypt password")
 
     user = UserService.query_user(email, password)
 
     if user and hasattr(user, 'is_active') and user.is_active == "0":
         return get_json_result(
             data=False,
-            code=settings.RetCode.FORBIDDEN,
+            code=RetCode.FORBIDDEN,
             message="This account has been disabled, please contact the administrator!",
         )
     elif user:
@@ -133,7 +129,7 @@ def login():
     else:
         return get_json_result(
             data=False,
-            code=settings.RetCode.AUTHENTICATION_ERROR,
+            code=RetCode.AUTHENTICATION_ERROR,
             message="Email and password do not match!",
         )
 
@@ -156,7 +152,7 @@ def get_login_channels():
         return get_json_result(data=channels)
     except Exception as e:
         logging.exception(e)
-        return get_json_result(data=[], message=f"Load channels failure, error: {str(e)}", code=settings.RetCode.EXCEPTION_ERROR)
+        return get_json_result(data=[], message=f"Load channels failure, error: {str(e)}", code=RetCode.EXCEPTION_ERROR)
 
 
 @manager.route("/login/<channel>", methods=["GET"])  # noqa: F821
@@ -540,7 +536,7 @@ def setting_user():
         if not check_password_hash(current_user.password, decrypt(request_data["password"])):
             return get_json_result(
                 data=False,
-                code=settings.RetCode.AUTHENTICATION_ERROR,
+                code=RetCode.AUTHENTICATION_ERROR,
                 message="Password error!",
             )
 
@@ -568,7 +564,7 @@ def setting_user():
         return get_json_result(data=True)
     except Exception as e:
         logging.exception(e)
-        return get_json_result(data=False, message="Update failure!", code=settings.RetCode.EXCEPTION_ERROR)
+        return get_json_result(data=False, message="Update failure!", code=RetCode.EXCEPTION_ERROR)
 
 
 @manager.route("/info", methods=["GET"])  # noqa: F821
@@ -698,7 +694,7 @@ def user_add():
         return get_json_result(
             data=False,
             message="User registration is disabled!",
-            code=settings.RetCode.OPERATING_ERROR,
+            code=RetCode.OPERATING_ERROR,
         )
 
     req = request.json
@@ -709,7 +705,7 @@ def user_add():
         return get_json_result(
             data=False,
             message=f"Invalid email address: {email_address}!",
-            code=settings.RetCode.OPERATING_ERROR,
+            code=RetCode.OPERATING_ERROR,
         )
 
     # Check if the email address is already used
@@ -717,7 +713,7 @@ def user_add():
         return get_json_result(
             data=False,
             message=f"Email: {email_address} has already registered!",
-            code=settings.RetCode.OPERATING_ERROR,
+            code=RetCode.OPERATING_ERROR,
         )
 
     # Construct user info data
@@ -752,7 +748,7 @@ def user_add():
         return get_json_result(
             data=False,
             message=f"User registration failure, error: {str(e)}",
-            code=settings.RetCode.EXCEPTION_ERROR,
+            code=RetCode.EXCEPTION_ERROR,
         )
 
 
@@ -852,11 +848,11 @@ def forget_get_captcha():
     """
     email = (request.args.get("email") or "")
     if not email:
-        return get_json_result(data=False, code=settings.RetCode.ARGUMENT_ERROR, message="email is required")
+        return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="email is required")
 
     users = UserService.query(email=email)
     if not users:
-        return get_json_result(data=False, code=settings.RetCode.DATA_ERROR, message="invalid email")
+        return get_json_result(data=False, code=RetCode.DATA_ERROR, message="invalid email")
 
     # Generate captcha text
     allowed = string.ascii_uppercase + string.digits
@@ -883,17 +879,17 @@ def forget_send_otp():
     captcha = (req.get("captcha") or "").strip()
 
     if not email or not captcha:
-        return get_json_result(data=False, code=settings.RetCode.ARGUMENT_ERROR, message="email and captcha required")
+        return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="email and captcha required")
 
     users = UserService.query(email=email)
     if not users:
-        return get_json_result(data=False, code=settings.RetCode.DATA_ERROR, message="invalid email")
+        return get_json_result(data=False, code=RetCode.DATA_ERROR, message="invalid email")
 
     stored_captcha = REDIS_CONN.get(captcha_key(email))
     if not stored_captcha:
-        return get_json_result(data=False, code=settings.RetCode.NOT_EFFECTIVE, message="invalid or expired captcha")
+        return get_json_result(data=False, code=RetCode.NOT_EFFECTIVE, message="invalid or expired captcha")
     if (stored_captcha or "").strip().lower() != captcha.lower():
-        return get_json_result(data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="invalid or expired captcha")
+        return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="invalid or expired captcha")
 
     # Delete captcha to prevent reuse
     REDIS_CONN.delete(captcha_key(email))
@@ -908,7 +904,7 @@ def forget_send_otp():
             elapsed = RESEND_COOLDOWN_SECONDS
         remaining = RESEND_COOLDOWN_SECONDS - elapsed
         if remaining > 0:
-            return get_json_result(data=False, code=settings.RetCode.NOT_EFFECTIVE, message=f"you still have to wait {remaining} seconds")
+            return get_json_result(data=False, code=RetCode.NOT_EFFECTIVE, message=f"you still have to wait {remaining} seconds")
 
     # Generate OTP (uppercase letters only) and store hashed
     otp = "".join(secrets.choice(string.ascii_uppercase) for _ in range(OTP_LENGTH))
@@ -933,9 +929,9 @@ def forget_send_otp():
                 ttl_min=ttl_min,
             )
         except Exception:
-            return get_json_result(data=False, code=settings.RetCode.SERVER_ERROR, message="failed to send email")
+            return get_json_result(data=False, code=RetCode.SERVER_ERROR, message="failed to send email")
         
-    return get_json_result(data=True, code=settings.RetCode.SUCCESS, message="verification passed, email sent")
+    return get_json_result(data=True, code=RetCode.SUCCESS, message="verification passed, email sent")
 
 
 @manager.route("/forget", methods=["POST"])  # noqa: F821
@@ -951,31 +947,31 @@ def forget():
     new_pwd2 = req.get("confirm_new_password")
 
     if not all([email, otp, new_pwd, new_pwd2]):
-        return get_json_result(data=False, code=settings.RetCode.ARGUMENT_ERROR, message="email, otp and passwords are required")
+        return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="email, otp and passwords are required")
 
     # For reset, passwords are provided as-is (no decrypt needed)
     if new_pwd != new_pwd2:
-        return get_json_result(data=False, code=settings.RetCode.ARGUMENT_ERROR, message="passwords do not match")
+        return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="passwords do not match")
 
     users = UserService.query(email=email)
     if not users:
-        return get_json_result(data=False, code=settings.RetCode.DATA_ERROR, message="invalid email")
+        return get_json_result(data=False, code=RetCode.DATA_ERROR, message="invalid email")
 
     user = users[0]
     # Verify OTP from Redis
     k_code, k_attempts, k_last, k_lock = otp_keys(email)
     if REDIS_CONN.get(k_lock):
-        return get_json_result(data=False, code=settings.RetCode.NOT_EFFECTIVE, message="too many attempts, try later")
+        return get_json_result(data=False, code=RetCode.NOT_EFFECTIVE, message="too many attempts, try later")
 
     stored = REDIS_CONN.get(k_code)
     if not stored:
-        return get_json_result(data=False, code=settings.RetCode.NOT_EFFECTIVE, message="expired otp")
+        return get_json_result(data=False, code=RetCode.NOT_EFFECTIVE, message="expired otp")
 
     try:
         stored_hash, salt_hex = str(stored).split(":", 1)
         salt = bytes.fromhex(salt_hex)
     except Exception:
-        return get_json_result(data=False, code=settings.RetCode.EXCEPTION_ERROR, message="otp storage corrupted")
+        return get_json_result(data=False, code=RetCode.EXCEPTION_ERROR, message="otp storage corrupted")
 
     # Case-insensitive verification: OTP generated uppercase
     calc = hash_code(otp.upper(), salt)
@@ -988,7 +984,7 @@ def forget():
         REDIS_CONN.set(k_attempts, attempts, OTP_TTL_SECONDS)
         if attempts >= ATTEMPT_LIMIT:
             REDIS_CONN.set(k_lock, int(time.time()), ATTEMPT_LOCK_SECONDS)
-        return get_json_result(data=False, code=settings.RetCode.AUTHENTICATION_ERROR, message="expired otp")
+        return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="expired otp")
 
     # Success: consume OTP and reset password
     REDIS_CONN.delete(k_code)
@@ -1000,7 +996,7 @@ def forget():
         UserService.update_user_password(user.id, new_pwd)
     except Exception as e:
         logging.exception(e)
-        return get_json_result(data=False, code=settings.RetCode.EXCEPTION_ERROR, message="failed to reset password")
+        return get_json_result(data=False, code=RetCode.EXCEPTION_ERROR, message="failed to reset password")
 
     # Auto login (reuse login flow)
     user.access_token = get_uuid()

api/constants.py

@@ -17,8 +17,6 @@ NAME_LENGTH_LIMIT = 2**10
 
 IMG_BASE64_PREFIX = "data:image/png;base64,"
 
-SERVICE_CONF = "service_conf.yaml"
-
 API_VERSION = "v1"
 RAG_FLOW_SERVICE_NAME = "ragflow"
 REQUEST_WAIT_SEC = 2

api/db/__init__.py

@@ -13,21 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-from enum import Enum
+
 from enum import IntEnum
 from strenum import StrEnum
 
 
-class StatusEnum(Enum):
-    VALID = "1"
-    INVALID = "0"
-
-
-class ActiveEnum(Enum):
-    ACTIVE = "1"
-    INACTIVE = "0"
-
-
 class UserTenantRole(StrEnum):
     OWNER = 'owner'
     ADMIN = 'admin'
@@ -56,76 +46,18 @@ class FileType(StrEnum):
 
 VALID_FILE_TYPES = {FileType.PDF, FileType.DOC, FileType.VISUAL, FileType.AURAL, FileType.VIRTUAL, FileType.FOLDER, FileType.OTHER}
 
-class LLMType(StrEnum):
-    CHAT = 'chat'
-    EMBEDDING = 'embedding'
-    SPEECH2TEXT = 'speech2text'
-    IMAGE2TEXT = 'image2text'
-    RERANK = 'rerank'
-    TTS    = 'tts'
 
-
-class ChatStyle(StrEnum):
-    CREATIVE = 'Creative'
-    PRECISE = 'Precise'
-    EVENLY = 'Evenly'
-    CUSTOM = 'Custom'
-
-
-class TaskStatus(StrEnum):
-    UNSTART = "0"
-    RUNNING = "1"
-    CANCEL = "2"
-    DONE = "3"
-    FAIL = "4"
-
-
-VALID_TASK_STATUS     = {TaskStatus.UNSTART, TaskStatus.RUNNING, TaskStatus.CANCEL, TaskStatus.DONE, TaskStatus.FAIL}
-
-
-class ParserType(StrEnum):
-    PRESENTATION = "presentation"
-    LAWS = "laws"
-    MANUAL = "manual"
-    PAPER = "paper"
-    RESUME = "resume"
-    BOOK = "book"
-    QA = "qa"
-    TABLE = "table"
-    NAIVE = "naive"
-    PICTURE = "picture"
-    ONE = "one"
-    AUDIO = "audio"
-    EMAIL = "email"
-    KG = "knowledge_graph"
-    TAG = "tag"
-
-
-class FileSource(StrEnum):
-    LOCAL = ""
-    KNOWLEDGEBASE = "knowledgebase"
-    S3 = "s3"
-
-
-class CanvasType(StrEnum):
-    ChatBot = "chatbot"
-    DocBot = "docbot"
+class InputType(StrEnum):
+    LOAD_STATE = "load_state"  # e.g. loading a current full state or a save state, such as from a file
+    POLL = "poll"  # e.g. calling an API to get all documents in the last hour
+    EVENT = "event"  # e.g. registered an endpoint as a listener, and processing connector events
+    SLIM_RETRIEVAL = "slim_retrieval"
 
 
 class CanvasCategory(StrEnum):
     Agent = "agent_canvas"
     DataFlow = "dataflow_canvas"
 
-VALID_CANVAS_CATEGORIES = {CanvasCategory.Agent, CanvasCategory.DataFlow}
-
-
-class MCPServerType(StrEnum):
-    SSE = "sse"
-    STREAMABLE_HTTP = "streamable-http"
-
-
-VALID_MCP_SERVER_TYPES = {MCPServerType.SSE, MCPServerType.STREAMABLE_HTTP}
-
 
 class PipelineTaskType(StrEnum):
     PARSE = "Parse"

api/db/db_models.py

@@ -21,6 +21,7 @@ import os
 import sys
 import time
 import typing
+from datetime import datetime, timezone
 from enum import Enum
 from functools import wraps
 
@@ -30,22 +31,15 @@ from peewee import InterfaceError, OperationalError, BigIntegerField, BooleanFie
 from playhouse.migrate import MySQLMigrator, PostgresqlMigrator, migrate
 from playhouse.pool import PooledMySQLDatabase, PooledPostgresqlDatabase
 
-from api import settings, utils
-from api.db import ParserType, SerializedType
-from api.utils.json import json_dumps, json_loads
+from api import utils
+from api.db import SerializedType
+from api.utils.json_encode import json_dumps, json_loads
 from api.utils.configs import deserialize_b64, serialize_b64
 
-
-def singleton(cls, *args, **kw):
-    instances = {}
-
-    def _singleton():
-        key = str(cls) + str(os.getpid())
-        if key not in instances:
-            instances[key] = cls(*args, **kw)
-        return instances[key]
-
-    return _singleton
+from common.time_utils import current_timestamp, timestamp_to_date, date_string_to_timestamp
+from common.decorator import singleton
+from common.constants import ParserType
+from common import settings
 
 
 CONTINUOUS_FIELD_TYPE = {IntegerField, FloatField, DateTimeField}
@@ -189,7 +183,7 @@ class BaseModel(Model):
                         for i, v in enumerate(f_v):
                             if isinstance(v, str) and f_n in auto_date_timestamp_field():
                                 # time type: %Y-%m-%d %H:%M:%S
-                                f_v[i] = utils.date_string_to_timestamp(v)
+                                f_v[i] = date_string_to_timestamp(v)
                         lt_value = f_v[0]
                         gt_value = f_v[1]
                         if lt_value is not None and gt_value is not None:
@@ -218,9 +212,9 @@ class BaseModel(Model):
     @classmethod
     def insert(cls, __data=None, **insert):
         if isinstance(__data, dict) and __data:
-            __data[cls._meta.combined["create_time"]] = utils.current_timestamp()
+            __data[cls._meta.combined["create_time"]] = current_timestamp()
         if insert:
-            insert["create_time"] = utils.current_timestamp()
+            insert["create_time"] = current_timestamp()
 
         return super().insert(__data, **insert)
 
@@ -231,11 +225,11 @@ class BaseModel(Model):
         if not normalized:
             return {}
 
-        normalized[cls._meta.combined["update_time"]] = utils.current_timestamp()
+        normalized[cls._meta.combined["update_time"]] = current_timestamp()
 
         for f_n in AUTO_DATE_TIMESTAMP_FIELD_PREFIX:
             if {f"{f_n}_time", f"{f_n}_date"}.issubset(cls._meta.combined.keys()) and cls._meta.combined[f"{f_n}_time"] in normalized and normalized[cls._meta.combined[f"{f_n}_time"]] is not None:
-                normalized[cls._meta.combined[f"{f_n}_date"]] = utils.timestamp_to_date(normalized[cls._meta.combined[f"{f_n}_time"]])
+                normalized[cls._meta.combined[f"{f_n}_date"]] = timestamp_to_date(normalized[cls._meta.combined[f"{f_n}_time"]])
 
         return normalized
 
@@ -331,9 +325,9 @@ class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):
                 # 08006: connection_failure
                 # 08003: connection_does_not_exist
                 # 08000: connection_exception
-                error_messages = ['connection', 'server closed', 'connection refused', 
+                error_messages = ['connection', 'server closed', 'connection refused',
                                 'no connection to the server', 'terminating connection']
-                
+
                 should_retry = any(msg in str(e).lower() for msg in error_messages)
 
                 if should_retry and attempt < self.max_retries:
@@ -366,7 +360,7 @@ class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):
             except (OperationalError, InterfaceError) as e:
                 error_messages = ['connection', 'server closed', 'connection refused',
                                 'no connection to the server', 'terminating connection']
-                
+
                 should_retry = any(msg in str(e).lower() for msg in error_messages)
 
                 if should_retry and attempt < self.max_retries:
@@ -377,6 +371,7 @@ class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):
                     time.sleep(self.retry_delay * (2 ** attempt))
                 else:
                     raise
+        return None
 
 
 class PooledDatabase(Enum):
@@ -394,7 +389,7 @@ class BaseDataBase:
     def __init__(self):
         database_config = settings.DATABASE.copy()
         db_name = database_config.pop("name")
-        
+
         pool_config = {
             'max_retries': 5,
             'retry_delay': 1,
@@ -711,6 +706,7 @@ class TenantLLM(DataBaseModel):
     api_base = CharField(max_length=255, null=True, help_text="API Base")
     max_tokens = IntegerField(default=8192, index=True)
     used_tokens = IntegerField(default=0, index=True)
+    status = CharField(max_length=1, null=False, help_text="is it validate(0: wasted, 1: validate)", default="1", index=True)
 
     def __str__(self):
         return self.llm_name
@@ -1044,6 +1040,76 @@ class PipelineOperationLog(DataBaseModel):
         db_table = "pipeline_operation_log"
 
 
+class Connector(DataBaseModel):
+    id = CharField(max_length=32, primary_key=True)
+    tenant_id = CharField(max_length=32, null=False, index=True)
+    name = CharField(max_length=128, null=False, help_text="Search name", index=False)
+    source = CharField(max_length=128, null=False, help_text="Data source", index=True)
+    input_type = CharField(max_length=128, null=False, help_text="poll/event/..", index=True)
+    config = JSONField(null=False, default={})
+    refresh_freq = IntegerField(default=0, index=False)
+    prune_freq = IntegerField(default=0, index=False)
+    timeout_secs = IntegerField(default=3600, index=False)
+    indexing_start = DateTimeField(null=True, index=True)
+    status = CharField(max_length=16, null=True, help_text="schedule", default="schedule", index=True)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        db_table = "connector"
+
+
+class Connector2Kb(DataBaseModel):
+    id = CharField(max_length=32, primary_key=True)
+    connector_id = CharField(max_length=32, null=False, index=True)
+    kb_id = CharField(max_length=32, null=False, index=True)
+
+    class Meta:
+        db_table = "connector2kb"
+
+
+class DateTimeTzField(CharField):
+    field_type = 'VARCHAR'
+
+    def db_value(self, value: datetime|None) -> str|None:
+        if value is not None:
+            if value.tzinfo is not None:
+                return value.isoformat()
+            else:
+                return value.replace(tzinfo=timezone.utc).isoformat()
+        return value
+
+    def python_value(self, value: str|None) -> datetime|None:
+        if value is not None:
+            dt = datetime.fromisoformat(value)
+            if dt.tzinfo is None:
+                import pytz
+                return dt.replace(tzinfo=pytz.UTC)
+            return dt
+        return value
+
+
+class SyncLogs(DataBaseModel):
+    id = CharField(max_length=32, primary_key=True)
+    connector_id = CharField(max_length=32, index=True)
+    status = CharField(max_length=128, null=False, help_text="Processing status", index=True)
+    from_beginning = CharField(max_length=1, null=True, help_text="", default="0", index=False)
+    new_docs_indexed = IntegerField(default=0, index=False)
+    total_docs_indexed = IntegerField(default=0, index=False)
+    docs_removed_from_index = IntegerField(default=0, index=False)
+    error_msg = TextField(null=False, help_text="process message", default="")
+    error_count = IntegerField(default=0, index=False)
+    full_exception_trace = TextField(null=True, help_text="process message", default="")
+    time_started = DateTimeField(null=True, index=True)
+    poll_range_start = DateTimeTzField(max_length=255, null=True, index=True)
+    poll_range_end = DateTimeTzField(max_length=255, null=True, index=True)
+    kb_id = CharField(max_length=32, null=False, index=True)
+
+    class Meta:
+        db_table = "sync_logs"
+
+
 def migrate_db():
     logging.disable(logging.ERROR)
     migrator = DatabaseMigrator[settings.DATABASE_TYPE.upper()].value(DB)
@@ -1212,4 +1278,8 @@ def migrate_db():
         migrate(migrator.alter_column_type("tenant_llm", "api_key", TextField(null=True, help_text="API KEY")))
     except Exception:
         pass
+    try:
+        migrate(migrator.add_column("tenant_llm", "status", CharField(max_length=1, null=False, help_text="is it validate(0: wasted, 1: validate)", default="1", index=True)))
+    except Exception:
+        pass
     logging.disable(logging.NOTSET)

api/db/db_utils.py

@@ -18,7 +18,7 @@ from functools import reduce
 
 from playhouse.pool import PooledMySQLDatabase
 
-from api.utils import current_timestamp, timestamp_to_date
+from common.time_utils import current_timestamp, timestamp_to_date
 
 from api.db.db_models import DB, DataBaseModel
 

api/db/init_data.py

@@ -20,7 +20,7 @@ import time
 import uuid
 from copy import deepcopy
 
-from api.db import LLMType, UserTenantRole
+from api.db import UserTenantRole
 from api.db.db_models import init_database_tables as init_web_db, LLMFactories, LLM, TenantLLM
 from api.db.services import UserService
 from api.db.services.canvas_service import CanvasTemplateService
@@ -29,8 +29,9 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.tenant_llm_service import LLMFactoriesService, TenantLLMService
 from api.db.services.llm_service import LLMService, LLMBundle, get_init_tenant_llm
 from api.db.services.user_service import TenantService, UserTenantService
-from api import settings
-from api.utils.file_utils import get_project_base_directory
+from common.constants import LLMType
+from common.file_utils import get_project_base_directory
+from common import settings
 from api.common.base64 import encode_to_base64
 
 

api/db/joint_services/user_account_service.py

@@ -16,9 +16,8 @@
 import logging
 import uuid
 
-from api import settings
 from api.utils.api_utils import group_by
-from api.db import FileType, UserTenantRole, ActiveEnum
+from api.db import FileType, UserTenantRole
 from api.db.services.api_service import APITokenService, API4ConversationService
 from api.db.services.canvas_service import UserCanvasService
 from api.db.services.conversation_service import ConversationService
@@ -35,9 +34,9 @@ from api.db.services.task_service import TaskService
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_canvas_version import UserCanvasVersionService
 from api.db.services.user_service import TenantService, UserService, UserTenantService
-from rag.utils.storage_factory import STORAGE_IMPL
 from rag.nlp import search
-
+from common.constants import ActiveEnum
+from common import settings
 
 def create_new_user(user_info: dict) -> dict:
     """
@@ -158,8 +157,8 @@ def delete_user_data(user_id: str) -> dict:
             if kb_ids:
                 # step1.1.1 delete files in storage, remove bucket
                 for kb_id in kb_ids:
-                    if STORAGE_IMPL.bucket_exists(kb_id):
-                        STORAGE_IMPL.remove_bucket(kb_id)
+                    if settings.STORAGE_IMPL.bucket_exists(kb_id):
+                        settings.STORAGE_IMPL.remove_bucket(kb_id)
                 done_msg += f"- Removed {len(kb_ids)} dataset's buckets.\n"
                 # step1.1.2 delete file and document info in db
                 doc_ids = DocumentService.get_all_doc_ids_by_kb_ids(kb_ids)
@@ -218,7 +217,7 @@ def delete_user_data(user_id: str) -> dict:
                     if created_files:
                         # step2.1.1.1 delete file in storage
                         for f in created_files:
-                            STORAGE_IMPL.rm(f.parent_id, f.location)
+                            settings.STORAGE_IMPL.rm(f.parent_id, f.location)
                         done_msg += f"- Deleted {len(created_files)} uploaded file.\n"
                         # step2.1.1.2 delete file record
                         file_delete_res = FileService.delete_by_ids([f.id for f in created_files])

api/db/services/api_service.py

@@ -19,7 +19,7 @@ import peewee
 
 from api.db.db_models import DB, API4Conversation, APIToken, Dialog
 from api.db.services.common_service import CommonService
-from api.utils import current_timestamp, datetime_format
+from common.time_utils import current_timestamp, datetime_format
 
 
 class APITokenService(CommonService):