Docs: Update version references to v0.23.0 in READMEs and docs (#12253)

### What problem does this PR solve?

- Update version tags in README files (including translations) from
v0.22.1 to v0.23.0
- Modify Docker image references and documentation to reflect new
version
- Update version badges and image descriptions
- Maintain consistency across all language variants of README files

### Type of change

- [x] Documentation Update

Co-authored-by: Jin Hai <haijin.chn@gmail.com>

.github/copilot-instructions.md

@@ -0,0 +1 @@
+Refer to [AGENTS.MD](../AGENTS.md) for all repo instructions.

.github/workflows/release.yml

@@ -3,11 +3,12 @@ name: release
 on:
   schedule:
     - cron: '0 13 * * *'  # This schedule runs every 13:00:00Z(21:00:00+08:00)
+  # https://github.com/orgs/community/discussions/26286?utm_source=chatgpt.com#discussioncomment-3251208
+  # "The create event does not support branch filter and tag filter."
   # The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags.
-  create:
+  push:
     tags:
       - "v*.*.*"                  # normal release
-      - "nightly"                 # the only one mutable tag
 
 # https://docs.github.com/en/actions/using-jobs/using-concurrency
 concurrency:
@@ -21,9 +22,9 @@ jobs:
       - name: Ensure workspace ownership
         run: echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
 
-      # https://github.com/actions/checkout/blob/v3/README.md
+      # https://github.com/actions/checkout/blob/v6/README.md
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}  # Use the secret as an environment variable
           fetch-depth: 0
@@ -31,12 +32,12 @@ jobs:
 
       - name: Prepare release body
         run: |
-          if [[ ${GITHUB_EVENT_NAME} == "create" ]]; then
+          if [[ ${GITHUB_EVENT_NAME} != "schedule" ]]; then
             RELEASE_TAG=${GITHUB_REF#refs/tags/}
-            if [[ ${RELEASE_TAG} == "nightly" ]]; then
-              PRERELEASE=true
-            else
+            if [[ ${RELEASE_TAG} == v* ]]; then
               PRERELEASE=false
+            else
+              PRERELEASE=true
             fi
             echo "Workflow triggered by create tag: ${RELEASE_TAG}"
           else
@@ -55,7 +56,7 @@ jobs:
           git fetch --tags
           if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
             # Determine if a given tag exists and matches a specific Git commit.
-            # actions/checkout@v4 fetch-tags doesn't work when triggered by schedule
+            # actions/checkout@v6 fetch-tags doesn't work when triggered by schedule
             if [ "$(git rev-parse -q --verify "refs/tags/${RELEASE_TAG}")" = "${GITHUB_SHA}" ]; then
               echo "mutable tag ${RELEASE_TAG} exists and matches ${GITHUB_SHA}"
             else
@@ -88,7 +89,7 @@ jobs:
       - name: Build and push image
         run: |
           sudo docker login --username infiniflow --password-stdin <<< ${{ secrets.DOCKERHUB_TOKEN }}
-          sudo docker build --build-arg NEED_MIRROR=1 -t infiniflow/ragflow:${RELEASE_TAG} -f Dockerfile .
+          sudo docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -t infiniflow/ragflow:${RELEASE_TAG} -f Dockerfile .
           sudo docker tag infiniflow/ragflow:${RELEASE_TAG} infiniflow/ragflow:latest
           sudo docker push infiniflow/ragflow:${RELEASE_TAG}
           sudo docker push infiniflow/ragflow:latest

.github/workflows/tests.yml

@@ -1,4 +1,6 @@
 name: tests
+permissions:
+  contents: read
 
 on:
   push:
@@ -34,9 +36,6 @@ jobs:
     if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci')) }}
     runs-on: [ "self-hosted", "ragflow-test" ]
     steps:
-      # https://github.com/hmarr/debug-action
-      #- uses: hmarr/debug-action@v2
-
       - name: Ensure workspace ownership
         run: |
           echo "Workflow triggered by ${{ github.event_name }}"
@@ -44,7 +43,7 @@ jobs:
 
       # https://github.com/actions/checkout/issues/1781
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
           fetch-depth: 0
@@ -129,7 +128,7 @@ jobs:
 
       - name: Run unit test
         run: |
-          uv sync --python 3.10 --group test --frozen
+          uv sync --python 3.12 --group test --frozen
           source .venv/bin/activate
           which pytest || echo "pytest not in PATH"
           echo "Start to run unit test"
@@ -141,7 +140,7 @@ jobs:
           RAGFLOW_IMAGE=infiniflow/ragflow:${GITHUB_RUN_ID}
           echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> ${GITHUB_ENV}
           sudo docker pull ubuntu:22.04
-          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t ${RAGFLOW_IMAGE} .
+          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} .
           if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
             export HTTP_API_TEST_LEVEL=p3
           else
@@ -201,34 +200,34 @@ jobs:
           echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV}
 
           sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} up -d
-          uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python --group test
+          uv sync --python 3.12 --only-group test --no-default-groups --frozen && uv pip install sdk/python --group test
 
       - name: Run sdk tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          source .venv/bin/activate && pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
+          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api 2>&1 | tee es_sdk_test.log
 
       - name: Run frontend api tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          source .venv/bin/activate && pytest -s --tb=short sdk/python/test/test_frontend_api/get_email.py sdk/python/test/test_frontend_api/test_dataset.py
+          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short sdk/python/test/test_frontend_api/get_email.py sdk/python/test/test_frontend_api/test_dataset.py 2>&1 | tee es_api_test.log
           
       - name: Run http api tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          source .venv/bin/activate && pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
+          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api 2>&1 | tee es_http_api_test.log
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
@@ -244,29 +243,29 @@ jobs:
       - name: Run sdk tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          source .venv/bin/activate && DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api
+          source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api 2>&1 | tee infinity_sdk_test.log
 
       - name: Run frontend api tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          source .venv/bin/activate && DOC_ENGINE=infinity pytest -s --tb=short sdk/python/test/test_frontend_api/get_email.py sdk/python/test/test_frontend_api/test_dataset.py
+          source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short sdk/python/test/test_frontend_api/get_email.py sdk/python/test/test_frontend_api/test_dataset.py 2>&1 | tee infinity_api_test.log
 
       - name: Run http api tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null; do
             echo "Waiting for service to be available..."
             sleep 5
           done
-          source .venv/bin/activate && DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api
+          source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api 2>&1 | tee infinity_http_api_test.log
 
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed

.gitignore

@@ -195,3 +195,6 @@ ragflow_cli.egg-info
 
 # Default backup dir
 backup
+
+
+.hypothesis

AGENTS.md

@@ -0,0 +1,110 @@
+# RAGFlow Project Instructions for GitHub Copilot
+
+This file provides context, build instructions, and coding standards for the RAGFlow project.
+It is structured to follow GitHub Copilot's [customization guidelines](https://docs.github.com/en/copilot/concepts/prompting/response-customization).
+
+## 1. Project Overview
+RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It is a full-stack application with a Python backend and a React/TypeScript frontend.
+
+- **Backend**: Python 3.10+ (Flask/Quart)
+- **Frontend**: TypeScript, React, UmiJS
+- **Architecture**: Microservices based on Docker.
+  - `api/`: Backend API server.
+  - `rag/`: Core RAG logic (indexing, retrieval).
+  - `deepdoc/`: Document parsing and OCR.
+  - `web/`: Frontend application.
+
+## 2. Directory Structure
+- `api/`: Backend API server (Flask/Quart).
+  - `apps/`: API Blueprints (Knowledge Base, Chat, etc.).
+  - `db/`: Database models and services.
+- `rag/`: Core RAG logic.
+  - `llm/`: LLM, Embedding, and Rerank model abstractions.
+- `deepdoc/`: Document parsing and OCR modules.
+- `agent/`: Agentic reasoning components.
+- `web/`: Frontend application (React + UmiJS).
+- `docker/`: Docker deployment configurations.
+- `sdk/`: Python SDK.
+- `test/`: Backend tests.
+
+## 3. Build Instructions
+
+### Backend (Python)
+The project uses **uv** for dependency management.
+
+1. **Setup Environment**:
+   ```bash
+   uv sync --python 3.12 --all-extras
+   uv run download_deps.py
+   ```
+
+2. **Run Server**:
+   - **Pre-requisite**: Start dependent services (MySQL, ES/Infinity, Redis, MinIO).
+     ```bash
+     docker compose -f docker/docker-compose-base.yml up -d
+     ```
+   - **Launch**:
+     ```bash
+     source .venv/bin/activate
+     export PYTHONPATH=$(pwd)
+     bash docker/launch_backend_service.sh
+     ```
+
+### Frontend (TypeScript/React)
+Located in `web/`.
+
+1. **Install Dependencies**:
+   ```bash
+   cd web
+   npm install
+   ```
+
+2. **Run Dev Server**:
+   ```bash
+   npm run dev
+   ```
+   Runs on port 8000 by default.
+
+### Docker Deployment
+To run the full stack using Docker:
+```bash
+cd docker
+docker compose -f docker-compose.yml up -d
+```
+
+## 4. Testing Instructions
+
+### Backend Tests
+- **Run All Tests**:
+  ```bash
+  uv run pytest
+  ```
+- **Run Specific Test**:
+  ```bash
+  uv run pytest test/test_api.py
+  ```
+
+### Frontend Tests
+- **Run Tests**:
+  ```bash
+  cd web
+  npm run test
+  ```
+
+## 5. Coding Standards & Guidelines
+- **Python Formatting**: Use `ruff` for linting and formatting.
+  ```bash
+  ruff check
+  ruff format
+  ```
+- **Frontend Linting**:
+  ```bash
+  cd web
+  npm run lint
+  ```
+- **Pre-commit**: Ensure pre-commit hooks are installed.
+  ```bash
+  pre-commit install
+  pre-commit run --all-files
+  ```
+

CLAUDE.md

@@ -45,7 +45,7 @@ RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on d
 ### Backend Development
 ```bash
 # Install Python dependencies
-uv sync --python 3.10 --all-extras
+uv sync --python 3.12 --all-extras
 uv run download_deps.py
 pre-commit install
 

Dockerfile

@@ -1,5 +1,5 @@
 # base stage
-FROM ubuntu:22.04 AS base
+FROM ubuntu:24.04 AS base
 USER root
 SHELL ["/bin/bash", "-c"]
 
@@ -33,36 +33,41 @@ ENV DEBIAN_FRONTEND=noninteractive
 # selenium:      libatk-bridge2.0-0                       chrome-linux64-121-0-6167-85
 # Building C extensions: libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev
 RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
+    apt update && \
+    apt --no-install-recommends install -y ca-certificates; \
     if [ "$NEED_MIRROR" == "1" ]; then \
-        sed -i 's|http://ports.ubuntu.com|http://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list; \
-        sed -i 's|http://archive.ubuntu.com|http://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list; \
+        sed -i 's|http://archive.ubuntu.com/ubuntu|https://mirrors.tuna.tsinghua.edu.cn/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources; \
+        sed -i 's|http://security.ubuntu.com/ubuntu|https://mirrors.tuna.tsinghua.edu.cn/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources; \
     fi; \
     rm -f /etc/apt/apt.conf.d/docker-clean && \
     echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache && \
     chmod 1777 /tmp && \
     apt update && \
-    apt --no-install-recommends install -y ca-certificates && \
-    apt update && \
     apt install -y libglib2.0-0 libglx-mesa0 libgl1 && \
     apt install -y pkg-config libicu-dev libgdiplus && \
     apt install -y default-jdk && \
     apt install -y libatk-bridge2.0-0 && \
     apt install -y libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev && \
     apt install -y libjemalloc-dev && \
-    apt install -y python3-pip pipx nginx unzip curl wget git vim less && \
+    apt install -y nginx unzip curl wget git vim less && \
     apt install -y ghostscript && \
     apt install -y pandoc && \
-    apt install -y texlive
+    apt install -y texlive && \
+    apt install -y fonts-freefont-ttf fonts-noto-cjk
 
-RUN if [ "$NEED_MIRROR" == "1" ]; then \
-        pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \
-        pip3 config set global.trusted-host pypi.tuna.tsinghua.edu.cn; \
+# Install uv
+RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/deps \
+    if [ "$NEED_MIRROR" == "1" ]; then \
         mkdir -p /etc/uv && \
-        echo "[[index]]" > /etc/uv/uv.toml && \
+        echo 'python-install-mirror = "https://registry.npmmirror.com/-/binary/python-build-standalone/"' > /etc/uv/uv.toml && \
+        echo '[[index]]' >> /etc/uv/uv.toml && \
         echo 'url = "https://pypi.tuna.tsinghua.edu.cn/simple"' >> /etc/uv/uv.toml && \
-        echo "default = true" >> /etc/uv/uv.toml; \
+        echo 'default = true' >> /etc/uv/uv.toml; \
     fi; \
-    pipx install uv
+    tar xzf /deps/uv-x86_64-unknown-linux-gnu.tar.gz \
+    && cp uv-x86_64-unknown-linux-gnu/* /usr/local/bin/ \
+    && rm -rf uv-x86_64-unknown-linux-gnu \
+    && uv python install 3.11
 
 ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
 ENV PATH=/root/.local/bin:$PATH
@@ -78,12 +83,12 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
 # A modern version of cargo is needed for the latest version of the Rust compiler.
 RUN apt update && apt install -y curl build-essential \
     && if [ "$NEED_MIRROR" == "1" ]; then \
-         # Use TUNA mirrors for rustup/rust dist files
+         # Use TUNA mirrors for rustup/rust dist files \
          export RUSTUP_DIST_SERVER="https://mirrors.tuna.tsinghua.edu.cn/rustup"; \
          export RUSTUP_UPDATE_ROOT="https://mirrors.tuna.tsinghua.edu.cn/rustup/rustup"; \
          echo "Using TUNA mirrors for Rustup."; \
        fi; \
-    # Force curl to use HTTP/1.1
+    # Force curl to use HTTP/1.1 \
     curl --proto '=https' --tlsv1.2 --http1.1 -sSf https://sh.rustup.rs | bash -s -- -y --profile minimal \
     && echo 'export PATH="/root/.cargo/bin:${PATH}"' >> /root/.bashrc
 
@@ -100,10 +105,10 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     apt update && \
     arch="$(uname -m)"; \
     if [ "$arch" = "arm64" ] || [ "$arch" = "aarch64" ]; then \
-        # ARM64 (macOS/Apple Silicon or Linux aarch64)
+        # ARM64 (macOS/Apple Silicon or Linux aarch64) \
         ACCEPT_EULA=Y apt install -y unixodbc-dev msodbcsql18; \
     else \
-        # x86_64 or others
+        # x86_64 or others \
         ACCEPT_EULA=Y apt install -y unixodbc-dev msodbcsql17; \
     fi || \
     { echo "Failed to install ODBC driver"; exit 1; }
@@ -147,7 +152,7 @@ RUN --mount=type=cache,id=ragflow_uv,target=/root/.cache/uv,sharing=locked \
     else \
         sed -i 's|pypi.tuna.tsinghua.edu.cn|pypi.org|g' uv.lock; \
     fi; \
-    uv sync --python 3.10 --frozen
+    uv sync --python 3.12 --frozen
 
 COPY web web
 COPY docs docs
@@ -187,6 +192,7 @@ COPY pyproject.toml uv.lock ./
 COPY mcp mcp
 COPY plugin plugin
 COPY common common
+COPY memory memory
 
 COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template
 COPY docker/entrypoint.sh ./

Dockerfile.deps

@@ -3,7 +3,7 @@
 FROM scratch
 
 # Copy resources downloaded via download_deps.py
-COPY chromedriver-linux64-121-0-6167-85 chrome-linux64-121-0-6167-85 cl100k_base.tiktoken libssl1.1_1.1.1f-1ubuntu2_amd64.deb libssl1.1_1.1.1f-1ubuntu2_arm64.deb tika-server-standard-3.0.0.jar tika-server-standard-3.0.0.jar.md5 libssl*.deb /
+COPY chromedriver-linux64-121-0-6167-85 chrome-linux64-121-0-6167-85 cl100k_base.tiktoken libssl1.1_1.1.1f-1ubuntu2_amd64.deb libssl1.1_1.1.1f-1ubuntu2_arm64.deb tika-server-standard-3.0.0.jar tika-server-standard-3.0.0.jar.md5 libssl*.deb uv-x86_64-unknown-linux-gnu.tar.gz /
 
 COPY nltk_data /nltk_data
 

README.md

@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.23.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -85,6 +85,7 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Latest Updates
 
+- 2025-12-26 Supports 'Memory' for AI agent.
 - 2025-11-19 Supports Gemini 3 Pro.
 - 2025-11-12 Supports data synchronization from Confluence, S3, Notion, Discord, Google Drive.
 - 2025-10-23 Supports MinerU & Docling as document parsing methods.
@@ -187,12 +188,12 @@ releases! 🌟
 > All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
 > If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
 
-> The command below downloads the `v0.22.1` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.22.1`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
+> The command below downloads the `v0.23.0` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.23.0`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
 
 ```bash
    $ cd ragflow/docker
   
-   # git checkout v0.22.1
+   # git checkout v0.23.0
    # Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
    # This step ensures the **entrypoint.sh** file in the code matches the Docker image version.
    
@@ -206,10 +207,10 @@ releases! 🌟
 
 > Note: Prior to `v0.22.0`, we provided both images with embedding models and slim images without embedding models. Details as follows:
 
-| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-| ----------------- | --------------- | --------------------- | ------------------------ |
-| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
-| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?        |
+|-------------------|-----------------|-----------------------|----------------|
+| v0.21.1           | &approx;9       | ✔️                    | Stable release |
+| v0.21.1-slim      | &approx;2       | ❌                     | Stable release |
 
 > Starting with `v0.22.0`, we ship only the slim edition and no longer append the **-slim** suffix to the image tag.
 
@@ -314,7 +315,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 # install RAGFlow dependent python modules
+   uv sync --python 3.12 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

README_id.md

@@ -22,7 +22,7 @@
         <img alt="Lencana Daring" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.23.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Rilis%20Terbaru" alt="Rilis Terbaru">
@@ -85,6 +85,7 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Pembaruan Terbaru
 
+- 2025-12-26 Mendukung 'Memori' untuk agen AI.
 - 2025-11-19 Mendukung Gemini 3 Pro.
 - 2025-11-12 Mendukung sinkronisasi data dari Confluence, S3, Notion, Discord, Google Drive.
 - 2025-10-23 Mendukung MinerU & Docling sebagai metode penguraian dokumen.
@@ -187,12 +188,12 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 > Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
 > Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
 
-> Perintah di bawah ini mengunduh edisi v0.22.1 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.22.1, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
+> Perintah di bawah ini mengunduh edisi v0.23.0 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.23.0, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
 
 ```bash
    $ cd ragflow/docker
    
-   # git checkout v0.22.1
+   # git checkout v0.23.0
    # Opsional: gunakan tag stabil (lihat releases: https://github.com/infiniflow/ragflow/releases)
    # This steps ensures the **entrypoint.sh** file in the code matches the Docker image version.
 
@@ -206,10 +207,10 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
 
 > Catatan: Sebelum `v0.22.0`, kami menyediakan image dengan model embedding dan image slim tanpa model embedding. Detailnya sebagai berikut:
 
-| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-| ----------------- | --------------- | --------------------- | ------------------------ |
-| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
-| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?        |
+|-------------------|-----------------|-----------------------|----------------|
+| v0.21.1           | &approx;9       | ✔️                    | Stable release |
+| v0.21.1-slim      | &approx;2       | ❌                     | Stable release |
 
 > Mulai dari `v0.22.0`, kami hanya menyediakan edisi slim dan tidak lagi menambahkan akhiran **-slim** pada tag image.
 
@@ -288,7 +289,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 # install RAGFlow dependent python modules
+   uv sync --python 3.12 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

README_ja.md

@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.23.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -66,7 +66,8 @@
 
 ## 🔥 最新情報
 
-- 2025-11-19 Gemini 3 Proをサポートしています
+- 2025-12-26 AIエージェントの「メモリ」機能をサポート。
+- 2025-11-19 Gemini 3 Proをサポートしています。
 - 2025-11-12 Confluence、S3、Notion、Discord、Google Drive からのデータ同期をサポートします。
 - 2025-10-23 ドキュメント解析方法として MinerU と Docling をサポートします。
 - 2025-10-15 オーケストレーションされたデータパイプラインのサポート。
@@ -167,12 +168,12 @@
 > 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
 > ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
 
-> 以下のコマンドは、RAGFlow Docker イメージの v0.22.1 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.22.1 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
+> 以下のコマンドは、RAGFlow Docker イメージの v0.23.0 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.23.0 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
 
 ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.22.1
+   # git checkout v0.23.0
    # 任意: 安定版タグを利用 (一覧: https://github.com/infiniflow/ragflow/releases)
    # この手順は、コード内の entrypoint.sh ファイルが Docker イメージのバージョンと一致していることを確認します。
 
@@ -186,10 +187,10 @@
 
 > 注意：`v0.22.0` より前のバージョンでは、embedding モデルを含むイメージと、embedding モデルを含まない slim イメージの両方を提供していました。詳細は以下の通りです：
 
-| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-| ----------------- | --------------- | --------------------- | ------------------------ |
-| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
-| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?        |
+|-------------------|-----------------|-----------------------|----------------|
+| v0.21.1           | &approx;9       | ✔️                    | Stable release |
+| v0.21.1-slim      | &approx;2       | ❌                     | Stable release |
 
 > `v0.22.0` 以降、当プロジェクトでは slim エディションのみを提供し、イメージタグに **-slim** サフィックスを付けなくなりました。
 
@@ -288,7 +289,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 # install RAGFlow dependent python modules
+   uv sync --python 3.12 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

README_ko.md

@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.23.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -67,6 +67,7 @@
 
 ## 🔥 업데이트
 
+- 2025-12-26 AI 에이전트의 '메모리' 기능 지원.
 - 2025-11-19 Gemini 3 Pro를 지원합니다.
 - 2025-11-12 Confluence, S3, Notion, Discord, Google Drive에서 데이터 동기화를 지원합니다.
 - 2025-10-23 문서 파싱 방법으로 MinerU 및 Docling을 지원합니다.
@@ -169,12 +170,12 @@
 > 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
 > ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
 
-   > 아래 명령어는 RAGFlow Docker 이미지의 v0.22.1 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.22.1과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
+   > 아래 명령어는 RAGFlow Docker 이미지의 v0.23.0 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.23.0과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
 
    ```bash
    $ cd ragflow/docker
    
-   # git checkout v0.22.1
+   # git checkout v0.23.0
    # Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
    # 이 단계는 코드의 entrypoint.sh 파일이 Docker 이미지 버전과 일치하도록 보장합니다.
 
@@ -188,10 +189,10 @@
 
 > 참고: `v0.22.0` 이전 버전에서는 embedding 모델이 포함된 이미지와 embedding 모델이 포함되지 않은 slim 이미지를 모두 제공했습니다. 자세한 내용은 다음과 같습니다:
 
-| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-| ----------------- | --------------- | --------------------- | ------------------------ |
-| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
-| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?        |
+|-------------------|-----------------|-----------------------|----------------|
+| v0.21.1           | &approx;9       | ✔️                    | Stable release |
+| v0.21.1-slim      | &approx;2       | ❌                     | Stable release |
 
 > `v0.22.0`부터는 slim 에디션만 배포하며 이미지 태그에 **-slim** 접미사를 더 이상 붙이지 않습니다.
 
@@ -283,7 +284,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 # install RAGFlow dependent python modules
+   uv sync --python 3.12 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

README_pt_br.md

@@ -22,7 +22,7 @@
         <img alt="Badge Estático" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.23.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Última%20Relese" alt="Última Versão">
@@ -86,6 +86,7 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
 ## 🔥 Últimas Atualizações
 
+- 26-12-2025 Suporte à função 'Memória' para agentes de IA.
 - 19-11-2025 Suporta Gemini 3 Pro.
 - 12-11-2025 Suporta a sincronização de dados do Confluence, S3, Notion, Discord e Google Drive.
 - 23-10-2025 Suporta MinerU e Docling como métodos de análise de documentos.
@@ -187,12 +188,12 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 > Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
 > Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
 
-    > O comando abaixo baixa a edição`v0.22.1` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.22.1`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
+    > O comando abaixo baixa a edição`v0.23.0` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.23.0`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
 
 ```bash
    $ cd ragflow/docker
    
-   # git checkout v0.22.1
+   # git checkout v0.23.0
    # Opcional: use uma tag estável (veja releases: https://github.com/infiniflow/ragflow/releases)
    # Esta etapa garante que o arquivo entrypoint.sh no código corresponda à versão da imagem do Docker.
 
@@ -206,10 +207,10 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
 
 > Nota: Antes da `v0.22.0`, fornecíamos imagens com modelos de embedding e imagens slim sem modelos de embedding. Detalhes a seguir:
 
-| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-| ----------------- | --------------- | --------------------- | ------------------------ |
-| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
-| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?        |
+|-------------------|-----------------|-----------------------|----------------|
+| v0.21.1           | &approx;9       | ✔️                    | Stable release |
+| v0.21.1-slim      | &approx;2       | ❌                     | Stable release |
 
 > A partir da `v0.22.0`, distribuímos apenas a edição slim e não adicionamos mais o sufixo **-slim** às tags das imagens.
 
@@ -305,7 +306,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 # instala os módulos Python dependentes do RAGFlow
+   uv sync --python 3.12 # instala os módulos Python dependentes do RAGFlow
    uv run download_deps.py
    pre-commit install
    ```

README_tzh.md

@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.23.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -85,15 +85,16 @@
 
 ## 🔥 近期更新
 
-- 2025-11-19 支援 Gemini 3 Pro.
+- 2025-12-26 支援AI代理的「記憶」功能。
+- 2025-11-19 支援 Gemini 3 Pro。
 - 2025-11-12 支援從 Confluence、S3、Notion、Discord、Google Drive 進行資料同步。
 - 2025-10-23 支援 MinerU 和 Docling 作為文件解析方法。
 - 2025-10-15 支援可編排的資料管道。
 - 2025-08-08 支援 OpenAI 最新的 GPT-5 系列模型。
-- 2025-08-01 支援 agentic workflow 和 MCP
+- 2025-08-01 支援 agentic workflow 和 MCP。
 - 2025-05-23 為 Agent 新增 Python/JS 程式碼執行器元件。
 - 2025-05-05 支援跨語言查詢。
-- 2025-03-19 PDF和DOCX中的圖支持用多模態大模型去解析得到描述.
+- 2025-03-19 PDF和DOCX中的圖支持用多模態大模型去解析得到描述。
 - 2024-12-18 升級了 DeepDoc 的文檔佈局分析模型。
 - 2024-08-22 支援用 RAG 技術實現從自然語言到 SQL 語句的轉換。
 
@@ -186,12 +187,12 @@
 > 所有 Docker 映像檔都是為 x86 平台建置的。目前，我們不提供 ARM64 平台的 Docker 映像檔。
 > 如果您使用的是 ARM64 平台，請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
 
-> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.22.1`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.22.1` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
+> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.23.0`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.23.0` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
 
 ```bash
    $ cd ragflow/docker
    
-   # git checkout v0.22.1
+   # git checkout v0.23.0
    # 可選：使用穩定版標籤（查看發佈：https://github.com/infiniflow/ragflow/releases）
    # 此步驟確保程式碼中的 entrypoint.sh 檔案與 Docker 映像版本一致。
 
@@ -205,10 +206,10 @@
 
 > 注意：在 `v0.22.0` 之前的版本，我們會同時提供包含 embedding 模型的映像和不含 embedding 模型的 slim 映像。具體如下：
 
-| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-| ----------------- | --------------- | --------------------- | ------------------------ |
-| v0.21.1           | &approx;9       | ✔️                    | Stable release           |
-| v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+| RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?        |
+|-------------------|-----------------|-----------------------|----------------|
+| v0.21.1           | &approx;9       | ✔️                    | Stable release |
+| v0.21.1-slim      | &approx;2       | ❌                     | Stable release |
 
 > 從 `v0.22.0` 開始，我們只發佈 slim 版本，並且不再在映像標籤後附加 **-slim** 後綴。
 
@@ -315,7 +316,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 # install RAGFlow dependent python modules
+   uv sync --python 3.12 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

README_zh.md

@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.23.0">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -85,7 +85,8 @@
 
 ## 🔥 近期更新
 
-- 2025-11-19 支持 Gemini 3 Pro.
+- 2025-12-26 支持AI代理的“记忆”功能。
+- 2025-11-19 支持 Gemini 3 Pro。
 - 2025-11-12 支持从 Confluence、S3、Notion、Discord、Google Drive 进行数据同步。
 - 2025-10-23 支持 MinerU 和 Docling 作为文档解析方法。
 - 2025-10-15 支持可编排的数据管道。
@@ -93,7 +94,7 @@
 - 2025-08-01 支持 agentic workflow 和 MCP。
 - 2025-05-23 Agent 新增 Python/JS 代码执行器组件。
 - 2025-05-05 支持跨语言查询。
-- 2025-03-19 PDF 和 DOCX 中的图支持用多模态大模型去解析得到描述.
+- 2025-03-19 PDF 和 DOCX 中的图支持用多模态大模型去解析得到描述。
 - 2024-12-18 升级了 DeepDoc 的文档布局分析模型。
 - 2024-08-22 支持用 RAG 技术实现从自然语言到 SQL 语句的转换。
 
@@ -187,12 +188,12 @@
 > 请注意，目前官方提供的所有 Docker 镜像均基于 x86 架构构建，并不提供基于 ARM64 的 Docker 镜像。
 > 如果你的操作系统是 ARM64 架构，请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
 
-   > 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.22.1`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.22.1` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
+   > 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.23.0`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.23.0` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
 
    ```bash
    $ cd ragflow/docker
    
-   # git checkout v0.22.1
+   # git checkout v0.23.0
    # 可选：使用稳定版本标签（查看发布：https://github.com/infiniflow/ragflow/releases）
    # 这一步确保代码中的 entrypoint.sh 文件与 Docker 镜像的版本保持一致。
 
@@ -206,10 +207,10 @@
    
    > 注意：在 `v0.22.0` 之前的版本，我们会同时提供包含 embedding 模型的镜像和不含 embedding 模型的 slim 镜像。具体如下：
 
-   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
-   | ----------------- | --------------- | --------------------- | ------------------------ |
-   | v0.21.1           | &approx;9       | ✔️                    | Stable release           |
-   | v0.21.1-slim      | &approx;2       | ❌                    | Stable release           |
+   | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?        |
+   |-------------------|-----------------|-----------------------|----------------|
+   | v0.21.1           | &approx;9       | ✔️                    | Stable release |
+   | v0.21.1-slim      | &approx;2       | ❌                     | Stable release |
 
    > 从 `v0.22.0` 开始，我们只发布 slim 版本，并且不再在镜像标签后附加 **-slim** 后缀。
 
@@ -315,7 +316,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.10 # install RAGFlow dependent python modules
+   uv sync --python 3.12 # install RAGFlow dependent python modules
    uv run download_deps.py
    pre-commit install
    ```

SECURITY.md

@@ -6,8 +6,8 @@ Use this section to tell people about which versions of your project are
 currently being supported with security updates.
 
 | Version | Supported          |
-| ------- | ------------------ |
-| <=0.7.0   | :white_check_mark: |
+|---------|--------------------|
+| <=0.7.0 | :white_check_mark: |
 
 ## Reporting a Vulnerability
 

admin/client/README.md

@@ -48,7 +48,7 @@ It consists of a server-side Service and a command-line client (CLI), both imple
 1.  Ensure the Admin Service is running.
 2.  Install ragflow-cli.
     ```bash
-    pip install ragflow-cli==0.22.1
+    pip install ragflow-cli==0.23.0
     ```
 3.  Launch the CLI client:
     ```bash

admin/client/admin_client.py

@@ -16,14 +16,14 @@
 
 import argparse
 import base64
-from cmd import Cmd
-
-from Cryptodome.PublicKey import RSA
-from Cryptodome.Cipher import PKCS1_v1_5 as Cipher_pkcs1_v1_5
-from typing import Dict, List, Any
-from lark import Lark, Transformer, Tree
-import requests
 import getpass
+from cmd import Cmd
+from typing import Any, Dict, List
+
+import requests
+from Cryptodome.Cipher import PKCS1_v1_5 as Cipher_pkcs1_v1_5
+from Cryptodome.PublicKey import RSA
+from lark import Lark, Transformer, Tree
 
 GRAMMAR = r"""
 start: command
@@ -141,7 +141,6 @@ NUMBER: /[0-9]+/
 
 
 class AdminTransformer(Transformer):
-
     def start(self, items):
         return items[0]
 
@@ -149,7 +148,7 @@ class AdminTransformer(Transformer):
         return items[0]
 
     def list_services(self, items):
-        result = {'type': 'list_services'}
+        result = {"type": "list_services"}
         return result
 
     def show_service(self, items):
@@ -236,11 +235,7 @@ class AdminTransformer(Transformer):
         action_list = items[1]
         resource = items[3]
         role_name = items[6]
-        return {
-            "type": "revoke_permission",
-            "role_name": role_name,
-            "resource": resource, "actions": action_list
-        }
+        return {"type": "revoke_permission", "role_name": role_name, "resource": resource, "actions": action_list}
 
     def alter_user_role(self, items):
         user_name = items[2]
@@ -264,12 +259,12 @@ class AdminTransformer(Transformer):
         # handle quoted parameter
         parsed_args = []
         for arg in args:
-            if hasattr(arg, 'value'):
+            if hasattr(arg, "value"):
                 parsed_args.append(arg.value)
             else:
                 parsed_args.append(str(arg))
 
-        return {'type': 'meta', 'command': command_name, 'args': parsed_args}
+        return {"type": "meta", "command": command_name, "args": parsed_args}
 
     def meta_command_name(self, items):
         return items[0]
@@ -279,22 +274,22 @@ class AdminTransformer(Transformer):
 
 
 def encrypt(input_string):
-    pub = '-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArq9XTUSeYr2+N1h3Afl/z8Dse/2yD0ZGrKwx+EEEcdsBLca9Ynmx3nIB5obmLlSfmskLpBo0UACBmB5rEjBp2Q2f3AG3Hjd4B+gNCG6BDaawuDlgANIhGnaTLrIqWrrcm4EMzJOnAOI1fgzJRsOOUEfaS318Eq9OVO3apEyCCt0lOQK6PuksduOjVxtltDav+guVAA068NrPYmRNabVKRNLJpL8w4D44sfth5RvZ3q9t+6RTArpEtc5sh5ChzvqPOzKGMXW83C95TxmXqpbK6olN4RevSfVjEAgCydH6HN6OhtOQEcnrU97r9H0iZOWwbw3pVrZiUkuRD1R56Wzs2wIDAQAB\n-----END PUBLIC KEY-----'
+    pub = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArq9XTUSeYr2+N1h3Afl/z8Dse/2yD0ZGrKwx+EEEcdsBLca9Ynmx3nIB5obmLlSfmskLpBo0UACBmB5rEjBp2Q2f3AG3Hjd4B+gNCG6BDaawuDlgANIhGnaTLrIqWrrcm4EMzJOnAOI1fgzJRsOOUEfaS318Eq9OVO3apEyCCt0lOQK6PuksduOjVxtltDav+guVAA068NrPYmRNabVKRNLJpL8w4D44sfth5RvZ3q9t+6RTArpEtc5sh5ChzvqPOzKGMXW83C95TxmXqpbK6olN4RevSfVjEAgCydH6HN6OhtOQEcnrU97r9H0iZOWwbw3pVrZiUkuRD1R56Wzs2wIDAQAB\n-----END PUBLIC KEY-----"
     pub_key = RSA.importKey(pub)
     cipher = Cipher_pkcs1_v1_5.new(pub_key)
-    cipher_text = cipher.encrypt(base64.b64encode(input_string.encode('utf-8')))
+    cipher_text = cipher.encrypt(base64.b64encode(input_string.encode("utf-8")))
     return base64.b64encode(cipher_text).decode("utf-8")
 
 
 def encode_to_base64(input_string):
-    base64_encoded = base64.b64encode(input_string.encode('utf-8'))
-    return base64_encoded.decode('utf-8')
+    base64_encoded = base64.b64encode(input_string.encode("utf-8"))
+    return base64_encoded.decode("utf-8")
 
 
 class AdminCLI(Cmd):
     def __init__(self):
         super().__init__()
-        self.parser = Lark(GRAMMAR, start='start', parser='lalr', transformer=AdminTransformer())
+        self.parser = Lark(GRAMMAR, start="start", parser="lalr", transformer=AdminTransformer())
         self.command_history = []
         self.is_interactive = False
         self.admin_account = "admin@ragflow.io"
@@ -312,7 +307,7 @@ class AdminCLI(Cmd):
             result = self.parse_command(command)
 
             if isinstance(result, dict):
-                if 'type' in result and result.get('type') == 'empty':
+                if "type" in result and result.get("type") == "empty":
                     return False
 
             self.execute_command(result)
@@ -320,7 +315,7 @@ class AdminCLI(Cmd):
             if isinstance(result, Tree):
                 return False
 
-            if result.get('type') == 'meta' and result.get('command') in ['q', 'quit', 'exit']:
+            if result.get("type") == "meta" and result.get("command") in ["q", "quit", "exit"]:
                 return True
 
         except KeyboardInterrupt:
@@ -338,7 +333,7 @@ class AdminCLI(Cmd):
 
     def parse_command(self, command_str: str) -> dict[str, str]:
         if not command_str.strip():
-            return {'type': 'empty'}
+            return {"type": "empty"}
 
         self.command_history.append(command_str)
 
@@ -346,12 +341,12 @@ class AdminCLI(Cmd):
             result = self.parser.parse(command_str)
             return result
         except Exception as e:
-            return {'type': 'error', 'message': f'Parse error: {str(e)}'}
+            return {"type": "error", "message": f"Parse error: {str(e)}"}
 
     def verify_admin(self, arguments: dict, single_command: bool):
-        self.host = arguments['host']
-        self.port = arguments['port']
-        print(f"Attempt to access ip: {self.host}, port: {self.port}")
+        self.host = arguments["host"]
+        self.port = arguments["port"]
+        print("Attempt to access server for admin login")
         url = f"http://{self.host}:{self.port}/api/v1/admin/login"
 
         attempt_count = 3
@@ -365,32 +360,28 @@ class AdminCLI(Cmd):
                 return False
 
             if single_command:
-                admin_passwd = arguments['password']
+                admin_passwd = arguments["password"]
             else:
                 admin_passwd = getpass.getpass(f"password for {self.admin_account}: ").strip()
             try:
                 self.admin_password = encrypt(admin_passwd)
-                response = self.session.post(url, json={'email': self.admin_account, 'password': self.admin_password})
+                response = self.session.post(url, json={"email": self.admin_account, "password": self.admin_password})
                 if response.status_code == 200:
                     res_json = response.json()
-                    error_code = res_json.get('code', -1)
+                    error_code = res_json.get("code", -1)
                     if error_code == 0:
-                        self.session.headers.update({
-                            'Content-Type': 'application/json',
-                            'Authorization': response.headers['Authorization'],
-                            'User-Agent': 'RAGFlow-CLI/0.22.1'
-                        })
+                        self.session.headers.update({"Content-Type": "application/json", "Authorization": response.headers["Authorization"], "User-Agent": "RAGFlow-CLI/0.23.0"})
                         print("Authentication successful.")
                         return True
                     else:
-                        error_message = res_json.get('message', 'Unknown error')
+                        error_message = res_json.get("message", "Unknown error")
                         print(f"Authentication failed: {error_message}, try again")
                         continue
                 else:
                     print(f"Bad response，status: {response.status_code}, password is wrong")
             except Exception as e:
                 print(str(e))
-                print(f"Can't access {self.host}, port: {self.port}")
+                print("Can't access server for admin login (connection failed)")
 
     def _format_service_detail_table(self, data):
         if isinstance(data, list):
@@ -403,10 +394,14 @@ class AdminCLI(Cmd):
         for k, v in data.items():
             # display latest status
             heartbeats = sorted(v, key=lambda x: x["now"], reverse=True)
-            task_executor_list.append({
-                "task_executor_name": k,
-                **heartbeats[0],
-            } if heartbeats else {"task_executor_name": k})
+            task_executor_list.append(
+                {
+                    "task_executor_name": k,
+                    **heartbeats[0],
+                }
+                if heartbeats
+                else {"task_executor_name": k}
+            )
         return task_executor_list
 
     def _print_table_simple(self, data):
@@ -422,12 +417,7 @@ class AdminCLI(Cmd):
         col_widths = {}
 
         def get_string_width(text):
-            half_width_chars = (
-                " !\"#$%&'()*+,-./0123456789:;<=>?@"
-                "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`"
-                "abcdefghijklmnopqrstuvwxyz{|}~"
-                "\t\n\r"
-            )
+            half_width_chars = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\t\n\r"
             width = 0
             for char in text:
                 if char in half_width_chars:
@@ -439,7 +429,7 @@ class AdminCLI(Cmd):
         for col in columns:
             max_width = get_string_width(str(col))
             for item in data:
-                value_len = get_string_width(str(item.get(col, '')))
+                value_len = get_string_width(str(item.get(col, "")))
                 if value_len > max_width:
                     max_width = value_len
             col_widths[col] = max(2, max_width)
@@ -457,16 +447,15 @@ class AdminCLI(Cmd):
         for item in data:
             row = "|"
             for col in columns:
-                value = str(item.get(col, ''))
+                value = str(item.get(col, ""))
                 if get_string_width(value) > col_widths[col]:
-                    value = value[:col_widths[col] - 3] + "..."
+                    value = value[: col_widths[col] - 3] + "..."
                 row += f" {value:<{col_widths[col] - (get_string_width(value) - len(value))}} |"
             print(row)
 
         print(separator)
 
     def run_interactive(self):
-
         self.is_interactive = True
         print("RAGFlow Admin command line interface - Type '\\?' for help, '\\q' to quit")
 
@@ -483,7 +472,7 @@ class AdminCLI(Cmd):
                 if isinstance(result, Tree):
                     continue
 
-                if result.get('type') == 'meta' and result.get('command') in ['q', 'quit', 'exit']:
+                if result.get("type") == "meta" and result.get("command") in ["q", "quit", "exit"]:
                     break
 
             except KeyboardInterrupt:
@@ -497,36 +486,30 @@ class AdminCLI(Cmd):
         self.execute_command(result)
 
     def parse_connection_args(self, args: List[str]) -> Dict[str, Any]:
-        parser = argparse.ArgumentParser(description='Admin CLI Client', add_help=False)
-        parser.add_argument('-h', '--host', default='localhost', help='Admin service host')
-        parser.add_argument('-p', '--port', type=int, default=9381, help='Admin service port')
-        parser.add_argument('-w', '--password', default='admin', type=str, help='Superuser password')
-        parser.add_argument('command', nargs='?', help='Single command')
+        parser = argparse.ArgumentParser(description="Admin CLI Client", add_help=False)
+        parser.add_argument("-h", "--host", default="localhost", help="Admin service host")
+        parser.add_argument("-p", "--port", type=int, default=9381, help="Admin service port")
+        parser.add_argument("-w", "--password", default="admin", type=str, help="Superuser password")
+        parser.add_argument("command", nargs="?", help="Single command")
         try:
             parsed_args, remaining_args = parser.parse_known_args(args)
             if remaining_args:
                 command = remaining_args[0]
-                return {
-                    'host': parsed_args.host,
-                    'port': parsed_args.port,
-                    'password': parsed_args.password,
-                    'command': command
-                }
+                return {"host": parsed_args.host, "port": parsed_args.port, "password": parsed_args.password, "command": command}
             else:
                 return {
-                    'host': parsed_args.host,
-                    'port': parsed_args.port,
+                    "host": parsed_args.host,
+                    "port": parsed_args.port,
                 }
         except SystemExit:
-            return {'error': 'Invalid connection arguments'}
+            return {"error": "Invalid connection arguments"}
 
     def execute_command(self, parsed_command: Dict[str, Any]):
-
         command_dict: dict
         if isinstance(parsed_command, Tree):
             command_dict = parsed_command.children[0]
         else:
-            if parsed_command['type'] == 'error':
+            if parsed_command["type"] == "error":
                 print(f"Error: {parsed_command['message']}")
                 return
             else:
@@ -534,56 +517,56 @@ class AdminCLI(Cmd):
 
         # print(f"Parsed command: {command_dict}")
 
-        command_type = command_dict['type']
+        command_type = command_dict["type"]
 
         match command_type:
-            case 'list_services':
+            case "list_services":
                 self._handle_list_services(command_dict)
-            case 'show_service':
+            case "show_service":
                 self._handle_show_service(command_dict)
-            case 'restart_service':
+            case "restart_service":
                 self._handle_restart_service(command_dict)
-            case 'shutdown_service':
+            case "shutdown_service":
                 self._handle_shutdown_service(command_dict)
-            case 'startup_service':
+            case "startup_service":
                 self._handle_startup_service(command_dict)
-            case 'list_users':
+            case "list_users":
                 self._handle_list_users(command_dict)
-            case 'show_user':
+            case "show_user":
                 self._handle_show_user(command_dict)
-            case 'drop_user':
+            case "drop_user":
                 self._handle_drop_user(command_dict)
-            case 'alter_user':
+            case "alter_user":
                 self._handle_alter_user(command_dict)
-            case 'create_user':
+            case "create_user":
                 self._handle_create_user(command_dict)
-            case 'activate_user':
+            case "activate_user":
                 self._handle_activate_user(command_dict)
-            case 'list_datasets':
+            case "list_datasets":
                 self._handle_list_datasets(command_dict)
-            case 'list_agents':
+            case "list_agents":
                 self._handle_list_agents(command_dict)
-            case 'create_role':
+            case "create_role":
                 self._create_role(command_dict)
-            case 'drop_role':
+            case "drop_role":
                 self._drop_role(command_dict)
-            case 'alter_role':
+            case "alter_role":
                 self._alter_role(command_dict)
-            case 'list_roles':
+            case "list_roles":
                 self._list_roles(command_dict)
-            case 'show_role':
+            case "show_role":
                 self._show_role(command_dict)
-            case 'grant_permission':
+            case "grant_permission":
                 self._grant_permission(command_dict)
-            case 'revoke_permission':
+            case "revoke_permission":
                 self._revoke_permission(command_dict)
-            case 'alter_user_role':
+            case "alter_user_role":
                 self._alter_user_role(command_dict)
-            case 'show_user_permission':
+            case "show_user_permission":
                 self._show_user_permission(command_dict)
-            case 'show_version':
+            case "show_version":
                 self._show_version(command_dict)
-            case 'meta':
+            case "meta":
                 self._handle_meta_command(command_dict)
             case _:
                 print(f"Command '{command_type}' would be executed with API")
@@ -591,29 +574,29 @@ class AdminCLI(Cmd):
     def _handle_list_services(self, command):
         print("Listing all services")
 
-        url = f'http://{self.host}:{self.port}/api/v1/admin/services'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/services"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
             print(f"Fail to get all services, code: {res_json['code']}, message: {res_json['message']}")
 
     def _handle_show_service(self, command):
-        service_id: int = command['number']
+        service_id: int = command["number"]
         print(f"Showing service: {service_id}")
 
-        url = f'http://{self.host}:{self.port}/api/v1/admin/services/{service_id}'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/services/{service_id}"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            res_data = res_json['data']
-            if 'status' in res_data and res_data['status'] == 'alive':
+            res_data = res_json["data"]
+            if "status" in res_data and res_data["status"] == "alive":
                 print(f"Service {res_data['service_name']} is alive, ")
-                if isinstance(res_data['message'], str):
-                    print(res_data['message'])
+                if isinstance(res_data["message"], str):
+                    print(res_data["message"])
                 else:
-                    data = self._format_service_detail_table(res_data['message'])
+                    data = self._format_service_detail_table(res_data["message"])
                     self._print_table_simple(data)
             else:
                 print(f"Service {res_data['service_name']} is down, {res_data['message']}")
@@ -621,47 +604,47 @@ class AdminCLI(Cmd):
             print(f"Fail to show service, code: {res_json['code']}, message: {res_json['message']}")
 
     def _handle_restart_service(self, command):
-        service_id: int = command['number']
+        service_id: int = command["number"]
         print(f"Restart service {service_id}")
 
     def _handle_shutdown_service(self, command):
-        service_id: int = command['number']
+        service_id: int = command["number"]
         print(f"Shutdown service {service_id}")
 
     def _handle_startup_service(self, command):
-        service_id: int = command['number']
+        service_id: int = command["number"]
         print(f"Startup service {service_id}")
 
     def _handle_list_users(self, command):
         print("Listing all users")
 
-        url = f'http://{self.host}:{self.port}/api/v1/admin/users'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/users"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
             print(f"Fail to get all users, code: {res_json['code']}, message: {res_json['message']}")
 
     def _handle_show_user(self, command):
-        username_tree: Tree = command['user_name']
+        username_tree: Tree = command["user_name"]
         user_name: str = username_tree.children[0].strip("'\"")
         print(f"Showing user: {user_name}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/users/{user_name}"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            table_data = res_json['data']
-            table_data.pop('avatar')
+            table_data = res_json["data"]
+            table_data.pop("avatar")
             self._print_table_simple(table_data)
         else:
             print(f"Fail to get user {user_name}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _handle_drop_user(self, command):
-        username_tree: Tree = command['user_name']
+        username_tree: Tree = command["user_name"]
         user_name: str = username_tree.children[0].strip("'\"")
         print(f"Drop user: {user_name}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/users/{user_name}"
         response = self.session.delete(url)
         res_json = response.json()
         if response.status_code == 200:
@@ -670,13 +653,13 @@ class AdminCLI(Cmd):
             print(f"Fail to drop user, code: {res_json['code']}, message: {res_json['message']}")
 
     def _handle_alter_user(self, command):
-        user_name_tree: Tree = command['user_name']
+        user_name_tree: Tree = command["user_name"]
         user_name: str = user_name_tree.children[0].strip("'\"")
-        password_tree: Tree = command['password']
+        password_tree: Tree = command["password"]
         password: str = password_tree.children[0].strip("'\"")
-        print(f"Alter user: {user_name}, password: {password}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/password'
-        response = self.session.put(url, json={'new_password': encrypt(password)})
+        print(f"Alter user: {user_name}, password: ******")
+        url = f"http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/password"
+        response = self.session.put(url, json={"new_password": encrypt(password)})
         res_json = response.json()
         if response.status_code == 200:
             print(res_json["message"])
@@ -684,32 +667,29 @@ class AdminCLI(Cmd):
             print(f"Fail to alter password, code: {res_json['code']}, message: {res_json['message']}")
 
     def _handle_create_user(self, command):
-        user_name_tree: Tree = command['user_name']
+        user_name_tree: Tree = command["user_name"]
         user_name: str = user_name_tree.children[0].strip("'\"")
-        password_tree: Tree = command['password']
+        password_tree: Tree = command["password"]
         password: str = password_tree.children[0].strip("'\"")
-        role: str = command['role']
-        print(f"Create user: {user_name}, password: {password}, role: {role}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/users'
-        response = self.session.post(
-            url,
-            json={'user_name': user_name, 'password': encrypt(password), 'role': role}
-        )
+        role: str = command["role"]
+        print(f"Create user: {user_name}, password: ******, role: {role}")
+        url = f"http://{self.host}:{self.port}/api/v1/admin/users"
+        response = self.session.post(url, json={"user_name": user_name, "password": encrypt(password), "role": role})
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
             print(f"Fail to create user {user_name}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _handle_activate_user(self, command):
-        user_name_tree: Tree = command['user_name']
+        user_name_tree: Tree = command["user_name"]
         user_name: str = user_name_tree.children[0].strip("'\"")
-        activate_tree: Tree = command['activate_status']
+        activate_tree: Tree = command["activate_status"]
         activate_status: str = activate_tree.children[0].strip("'\"")
-        if activate_status.lower() in ['on', 'off']:
+        if activate_status.lower() in ["on", "off"]:
             print(f"Alter user {user_name} activate status, turn {activate_status.lower()}.")
-            url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/activate'
-            response = self.session.put(url, json={'activate_status': activate_status})
+            url = f"http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/activate"
+            response = self.session.put(url, json={"activate_status": activate_status})
             res_json = response.json()
             if response.status_code == 200:
                 print(res_json["message"])
@@ -719,202 +699,182 @@ class AdminCLI(Cmd):
             print(f"Unknown activate status: {activate_status}.")
 
     def _handle_list_datasets(self, command):
-        username_tree: Tree = command['user_name']
+        username_tree: Tree = command["user_name"]
         user_name: str = username_tree.children[0].strip("'\"")
         print(f"Listing all datasets of user: {user_name}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/datasets'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/datasets"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            table_data = res_json['data']
+            table_data = res_json["data"]
             for t in table_data:
-                t.pop('avatar')
+                t.pop("avatar")
             self._print_table_simple(table_data)
         else:
             print(f"Fail to get all datasets of {user_name}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _handle_list_agents(self, command):
-        username_tree: Tree = command['user_name']
+        username_tree: Tree = command["user_name"]
         user_name: str = username_tree.children[0].strip("'\"")
         print(f"Listing all agents of user: {user_name}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/agents'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/users/{user_name}/agents"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            table_data = res_json['data']
+            table_data = res_json["data"]
             for t in table_data:
-                t.pop('avatar')
+                t.pop("avatar")
             self._print_table_simple(table_data)
         else:
             print(f"Fail to get all agents of {user_name}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _create_role(self, command):
-        role_name_tree: Tree = command['role_name']
+        role_name_tree: Tree = command["role_name"]
         role_name: str = role_name_tree.children[0].strip("'\"")
-        desc_str: str = ''
-        if 'description' in command:
-            desc_tree: Tree = command['description']
+        desc_str: str = ""
+        if "description" in command:
+            desc_tree: Tree = command["description"]
             desc_str = desc_tree.children[0].strip("'\"")
 
         print(f"create role name: {role_name}, description: {desc_str}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/roles'
-        response = self.session.post(
-            url,
-            json={'role_name': role_name, 'description': desc_str}
-        )
+        url = f"http://{self.host}:{self.port}/api/v1/admin/roles"
+        response = self.session.post(url, json={"role_name": role_name, "description": desc_str})
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
             print(f"Fail to create role {role_name}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _drop_role(self, command):
-        role_name_tree: Tree = command['role_name']
+        role_name_tree: Tree = command["role_name"]
         role_name: str = role_name_tree.children[0].strip("'\"")
         print(f"drop role name: {role_name}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name}'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/roles/{role_name}"
         response = self.session.delete(url)
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
             print(f"Fail to drop role {role_name}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _alter_role(self, command):
-        role_name_tree: Tree = command['role_name']
+        role_name_tree: Tree = command["role_name"]
         role_name: str = role_name_tree.children[0].strip("'\"")
-        desc_tree: Tree = command['description']
+        desc_tree: Tree = command["description"]
         desc_str: str = desc_tree.children[0].strip("'\"")
 
         print(f"alter role name: {role_name}, description: {desc_str}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name}'
-        response = self.session.put(
-            url,
-            json={'description': desc_str}
-        )
+        url = f"http://{self.host}:{self.port}/api/v1/admin/roles/{role_name}"
+        response = self.session.put(url, json={"description": desc_str})
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
-            print(
-                f"Fail to update role {role_name} with description: {desc_str}, code: {res_json['code']}, message: {res_json['message']}")
+            print(f"Fail to update role {role_name} with description: {desc_str}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _list_roles(self, command):
         print("Listing all roles")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/roles'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/roles"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
             print(f"Fail to list roles, code: {res_json['code']}, message: {res_json['message']}")
 
     def _show_role(self, command):
-        role_name_tree: Tree = command['role_name']
+        role_name_tree: Tree = command["role_name"]
         role_name: str = role_name_tree.children[0].strip("'\"")
         print(f"show role: {role_name}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name}/permission'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/roles/{role_name}/permission"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
             print(f"Fail to list roles, code: {res_json['code']}, message: {res_json['message']}")
 
     def _grant_permission(self, command):
-        role_name_tree: Tree = command['role_name']
+        role_name_tree: Tree = command["role_name"]
         role_name_str: str = role_name_tree.children[0].strip("'\"")
-        resource_tree: Tree = command['resource']
+        resource_tree: Tree = command["resource"]
         resource_str: str = resource_tree.children[0].strip("'\"")
-        action_tree_list: list = command['actions']
+        action_tree_list: list = command["actions"]
         actions: list = []
         for action_tree in action_tree_list:
             action_str: str = action_tree.children[0].strip("'\"")
             actions.append(action_str)
         print(f"grant role_name: {role_name_str}, resource: {resource_str}, actions: {actions}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name_str}/permission'
-        response = self.session.post(
-            url,
-            json={'actions': actions, 'resource': resource_str}
-        )
+        url = f"http://{self.host}:{self.port}/api/v1/admin/roles/{role_name_str}/permission"
+        response = self.session.post(url, json={"actions": actions, "resource": resource_str})
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
-            print(
-                f"Fail to grant role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
+            print(f"Fail to grant role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _revoke_permission(self, command):
-        role_name_tree: Tree = command['role_name']
+        role_name_tree: Tree = command["role_name"]
         role_name_str: str = role_name_tree.children[0].strip("'\"")
-        resource_tree: Tree = command['resource']
+        resource_tree: Tree = command["resource"]
         resource_str: str = resource_tree.children[0].strip("'\"")
-        action_tree_list: list = command['actions']
+        action_tree_list: list = command["actions"]
         actions: list = []
         for action_tree in action_tree_list:
             action_str: str = action_tree.children[0].strip("'\"")
             actions.append(action_str)
         print(f"revoke role_name: {role_name_str}, resource: {resource_str}, actions: {actions}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/roles/{role_name_str}/permission'
-        response = self.session.delete(
-            url,
-            json={'actions': actions, 'resource': resource_str}
-        )
+        url = f"http://{self.host}:{self.port}/api/v1/admin/roles/{role_name_str}/permission"
+        response = self.session.delete(url, json={"actions": actions, "resource": resource_str})
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
-            print(
-                f"Fail to revoke role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
+            print(f"Fail to revoke role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _alter_user_role(self, command):
-        role_name_tree: Tree = command['role_name']
+        role_name_tree: Tree = command["role_name"]
         role_name_str: str = role_name_tree.children[0].strip("'\"")
-        user_name_tree: Tree = command['user_name']
+        user_name_tree: Tree = command["user_name"]
         user_name_str: str = user_name_tree.children[0].strip("'\"")
         print(f"alter_user_role user_name: {user_name_str}, role_name: {role_name_str}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name_str}/role'
-        response = self.session.put(
-            url,
-            json={'role_name': role_name_str}
-        )
+        url = f"http://{self.host}:{self.port}/api/v1/admin/users/{user_name_str}/role"
+        response = self.session.put(url, json={"role_name": role_name_str})
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
-            print(
-                f"Fail to alter user: {user_name_str} to role {role_name_str}, code: {res_json['code']}, message: {res_json['message']}")
+            print(f"Fail to alter user: {user_name_str} to role {role_name_str}, code: {res_json['code']}, message: {res_json['message']}")
 
     def _show_user_permission(self, command):
-        user_name_tree: Tree = command['user_name']
+        user_name_tree: Tree = command["user_name"]
         user_name_str: str = user_name_tree.children[0].strip("'\"")
         print(f"show_user_permission user_name: {user_name_str}")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/users/{user_name_str}/permission'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/users/{user_name_str}/permission"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
-            print(
-                f"Fail to show user: {user_name_str} permission, code: {res_json['code']}, message: {res_json['message']}")
+            print(f"Fail to show user: {user_name_str} permission, code: {res_json['code']}, message: {res_json['message']}")
 
     def _show_version(self, command):
         print("show_version")
-        url = f'http://{self.host}:{self.port}/api/v1/admin/version'
+        url = f"http://{self.host}:{self.port}/api/v1/admin/version"
         response = self.session.get(url)
         res_json = response.json()
         if response.status_code == 200:
-            self._print_table_simple(res_json['data'])
+            self._print_table_simple(res_json["data"])
         else:
             print(f"Fail to show version, code: {res_json['code']}, message: {res_json['message']}")
 
     def _handle_meta_command(self, command):
-        meta_command = command['command']
-        args = command.get('args', [])
+        meta_command = command["command"]
+        args = command.get("args", [])
 
-        if meta_command in ['?', 'h', 'help']:
+        if meta_command in ["?", "h", "help"]:
             self.show_help()
-        elif meta_command in ['q', 'quit', 'exit']:
+        elif meta_command in ["q", "quit", "exit"]:
             print("Goodbye!")
         else:
             print(f"Meta command '{meta_command}' with args {args}")
@@ -950,17 +910,17 @@ def main():
     cli = AdminCLI()
 
     args = cli.parse_connection_args(sys.argv)
-    if 'error' in args:
-        print(f"Error: {args['error']}")
+    if "error" in args:
+        print("Error: Invalid connection arguments")
         return
 
-    if 'command' in args:
-        if 'password' not in args:
+    if "command" in args:
+        if "password" not in args:
             print("Error: password is missing")
             return
         if cli.verify_admin(args, single_command=True):
-            command: str = args['command']
-            print(f"Run single command: {command}")
+            command: str = args["command"]
+            # print(f"Run single command: {command}")
             cli.run_single_command(command)
     else:
         if cli.verify_admin(args, single_command=False):
@@ -974,5 +934,5 @@ def main():
             cli.cmdloop()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

admin/client/pyproject.toml

@@ -1,11 +1,11 @@
 [project]
 name = "ragflow-cli"
-version = "0.22.1"
+version = "0.23.0"
 description = "Admin Service's client of [RAGFlow](https://github.com/infiniflow/ragflow). The Admin Service provides user management and system monitoring. "
 authors = [{ name = "Lynn", email = "lynn_inf@hotmail.com" }]
 license = { text = "Apache License, Version 2.0" }
 readme = "README.md"
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.12,<3.15"
 dependencies = [
     "requests>=2.30.0,<3.0.0",
     "beartype>=0.20.0,<1.0.0",

admin/client/uv.lock

@@ -196,7 +196,7 @@ wheels = [
 
 [[package]]
 name = "ragflow-cli"
-version = "0.22.1"
+version = "0.23.0"
 source = { virtual = "." }
 dependencies = [
     { name = "beartype" },

admin/server/auth.py

@@ -176,11 +176,11 @@ def login_verify(f):
                     "message": "Access denied",
                     "data": None
                 }), 200
-        except Exception as e:
-            error_msg = str(e)
+        except Exception:
+            logging.exception("An error occurred during admin login verification.")
             return jsonify({
                 "code": 500,
-                "message": error_msg
+                "message": "An internal server error occurred."
             }), 200
 
         return f(*args, **kwargs)

admin/server/routes.py

@@ -29,6 +29,11 @@ from common.versions import get_ragflow_version
 admin_bp = Blueprint('admin', __name__, url_prefix='/api/v1/admin')
 
 
+@admin_bp.route('/ping', methods=['GET'])
+def ping():
+    return success_response('PONG')
+
+
 @admin_bp.route('/login', methods=['POST'])
 def login():
     if not request.json:

agent/__init__.py

@@ -13,6 +13,3 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-
-# from beartype.claw import beartype_this_package
-# beartype_this_package()

agent/canvas.py

@@ -160,7 +160,7 @@ class Graph:
         return self._tenant_id
 
     def get_value_with_variable(self,value: str) -> Any:
-        pat = re.compile(r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z0-9_.]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*")
+        pat = re.compile(r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z0-9_.-]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*")
         out_parts = []
         last = 0
 
@@ -278,7 +278,7 @@ class Graph:
 
 class Canvas(Graph):
 
-    def __init__(self, dsl: str, tenant_id=None, task_id=None):
+    def __init__(self, dsl: str, tenant_id=None, task_id=None, canvas_id=None):
         self.globals = {
             "sys.query": "",
             "sys.user_id": tenant_id,
@@ -287,6 +287,7 @@ class Canvas(Graph):
         }
         self.variables = {}
         super().__init__(dsl, tenant_id, task_id)
+        self._id = canvas_id
 
     def load(self):
         super().load()
@@ -368,8 +369,13 @@ class Canvas(Graph):
 
         if kwargs.get("webhook_payload"):
             for k, cpn in self.components.items():
-                if self.components[k]["obj"].component_name.lower() == "webhook":
-                    for kk, vv in kwargs["webhook_payload"].items():
+                if self.components[k]["obj"].component_name.lower() == "begin"  and self.components[k]["obj"]._param.mode == "Webhook":
+                    payload = kwargs.get("webhook_payload", {})
+                    if "input" in payload:
+                        self.components[k]["obj"].set_input_value("request", payload["input"])
+                    for kk, vv in payload.items():
+                        if kk == "input":
+                            continue
                         self.components[k]["obj"].set_output(kk, vv)
 
         for k in kwargs.keys():
@@ -478,7 +484,7 @@ class Canvas(Graph):
                 })
             await _run_batch(idx, to)
             to = len(self.path)
-            # post processing of components invocation
+            # post-processing of components invocation
             for i in range(idx, to):
                 cpn = self.get_component(self.path[i])
                 cpn_obj = self.get_component_obj(self.path[i])
@@ -534,10 +540,14 @@ class Canvas(Graph):
                         yield decorate("message", {"content": cpn_obj.output("content")})
                         cite = re.search(r"\[ID:[ 0-9]+\]",  cpn_obj.output("content"))
 
-                    if isinstance(cpn_obj.output("attachment"), tuple):
-                        yield decorate("message", {"attachment": cpn_obj.output("attachment")})
-
-                    yield decorate("message_end", {"reference": self.get_reference() if cite else None})
+                    message_end = {}
+                    if cpn_obj.get_param("status"):
+                        message_end["status"] = cpn_obj.get_param("status")
+                    if isinstance(cpn_obj.output("attachment"), dict):
+                        message_end["attachment"] = cpn_obj.output("attachment")
+                    if cite:
+                        message_end["reference"] = self.get_reference()
+                    yield decorate("message_end", message_end)
 
                     while partials:
                         _cpn_obj = self.get_component_obj(partials[0])
@@ -712,6 +722,9 @@ class Canvas(Graph):
     def get_mode(self):
         return self.components["begin"]["obj"]._param.mode
 
+    def get_sys_query(self):
+        return self.globals.get("sys.query", "")
+
     def set_global_param(self, **kwargs):
         self.globals.update(kwargs)
 

agent/component/agent_with_tools.py

@@ -29,8 +29,8 @@ from api.db.services.llm_service import LLMBundle
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.mcp_server_service import MCPServerService
 from common.connection_utils import timeout
-from rag.prompts.generator import next_step_async, COMPLETE_TASK, analyze_task_async, \
-    citation_prompt, reflect_async, kb_prompt, citation_plus, full_question, message_fit_in, structured_output_prompt
+from rag.prompts.generator import next_step_async, COMPLETE_TASK, \
+    citation_prompt, kb_prompt, citation_plus, full_question, message_fit_in, structured_output_prompt
 from common.mcp_tool_call_conn import MCPToolCallSession, mcp_tool_metadata_to_openai_tool
 from agent.component.llm import LLMParam, LLM
 
@@ -84,9 +84,11 @@ class Agent(LLM, ToolBase):
     def __init__(self, canvas, id, param: LLMParam):
         LLM.__init__(self, canvas, id, param)
         self.tools = {}
-        for cpn in self._param.tools:
+        for idx, cpn in enumerate(self._param.tools):
             cpn = self._load_tool_obj(cpn)
-            self.tools[cpn.get_meta()["function"]["name"]] = cpn
+            original_name = cpn.get_meta()["function"]["name"]
+            indexed_name = f"{original_name}_{idx}"
+            self.tools[indexed_name] = cpn
 
         self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), TenantLLMService.llm_id2llm_type(self._param.llm_id), self._param.llm_id,
                                   max_retries=self._param.max_retries,
@@ -94,7 +96,12 @@ class Agent(LLM, ToolBase):
                                   max_rounds=self._param.max_rounds,
                                   verbose_tool_use=True
                                   )
-        self.tool_meta = [v.get_meta() for _,v in self.tools.items()]
+        self.tool_meta = []
+        for indexed_name, tool_obj in self.tools.items():
+            original_meta = tool_obj.get_meta()
+            indexed_meta = deepcopy(original_meta)
+            indexed_meta["function"]["name"] = indexed_name
+            self.tool_meta.append(indexed_meta)
 
         for mcp in self._param.mcp:
             _, mcp_server = MCPServerService.get_by_id(mcp["mcp_id"])
@@ -108,7 +115,8 @@ class Agent(LLM, ToolBase):
 
     def _load_tool_obj(self, cpn: dict) -> object:
         from agent.component import component_class
-        param = component_class(cpn["component_name"] + "Param")()
+        tool_name = cpn["component_name"]
+        param = component_class(tool_name + "Param")()
         param.update(cpn["params"])
         try:
             param.check()
@@ -202,7 +210,7 @@ class Agent(LLM, ToolBase):
         _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97))
         use_tools = []
         ans = ""
-        async for delta_ans, _tk in self._react_with_tools_streamly_async(prompt, msg, use_tools, user_defined_prompt,schema_prompt=schema_prompt):
+        async for delta_ans, _tk in self._react_with_tools_streamly_async_simple(prompt, msg, use_tools, user_defined_prompt,schema_prompt=schema_prompt):
             if self.check_if_canceled("Agent processing"):
                 return
             ans += delta_ans
@@ -246,7 +254,7 @@ class Agent(LLM, ToolBase):
         _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97))
         answer_without_toolcall = ""
         use_tools = []
-        async for delta_ans, _ in self._react_with_tools_streamly_async(prompt, msg, use_tools, user_defined_prompt):
+        async for delta_ans, _ in self._react_with_tools_streamly_async_simple(prompt, msg, use_tools, user_defined_prompt):
             if self.check_if_canceled("Agent streaming"):
                 return
 
@@ -264,18 +272,36 @@ class Agent(LLM, ToolBase):
         if use_tools:
             self.set_output("use_tools", use_tools)
 
-    async def _react_with_tools_streamly_async(self, prompt, history: list[dict], use_tools, user_defined_prompt={}, schema_prompt: str = ""):
+    async def _react_with_tools_streamly_async_simple(self, prompt, history: list[dict], use_tools, user_defined_prompt={}, schema_prompt: str = ""):
         token_count = 0
         tool_metas = self.tool_meta
         hist = deepcopy(history)
         last_calling = ""
         if len(hist) > 3:
             st = timer()
-            user_request = await asyncio.to_thread(full_question, messages=history, chat_mdl=self.chat_mdl)
+            user_request = await full_question(messages=history, chat_mdl=self.chat_mdl)
             self.callback("Multi-turn conversation optimization", {}, user_request, elapsed_time=timer()-st)
         else:
             user_request = history[-1]["content"]
 
+        def build_task_desc(prompt: str, user_request: str, user_defined_prompt: dict | None = None) -> str:
+            """Build a minimal task_desc by concatenating prompt, query, and tool schemas."""
+            user_defined_prompt = user_defined_prompt or {}
+
+            task_desc = (
+                "### Agent Prompt\n"
+                f"{prompt}\n\n"
+                "### User Request\n"
+                f"{user_request}\n\n"
+            )
+
+            if user_defined_prompt:
+                udp_json = json.dumps(user_defined_prompt, ensure_ascii=False, indent=2)
+                task_desc += "\n### User Defined Prompts\n" + udp_json + "\n"
+
+            return task_desc
+
+
         async def use_tool_async(name, args):
             nonlocal hist, use_tools, last_calling
             logging.info(f"{last_calling=} == {name=}")
@@ -286,9 +312,6 @@ class Agent(LLM, ToolBase):
                 "arguments": args,
                 "results": tool_response
             })
-            # self.callback("add_memory", {}, "...")
-            #self.add_memory(hist[-2]["content"], hist[-1]["content"], name, args, str(tool_response), user_defined_prompt)
-
             return name, tool_response
 
         async def complete():
@@ -309,7 +332,7 @@ class Agent(LLM, ToolBase):
             if len(hist) > 12:
                 _hist = [hist[0], hist[1], *hist[-10:]]
             entire_txt = ""
-            async for delta_ans in self._generate_streamly_async(_hist):
+            async for delta_ans in self._generate_streamly(_hist):
                 if not need2cite or cited:
                     yield delta_ans, 0
                 entire_txt += delta_ans
@@ -326,6 +349,21 @@ class Agent(LLM, ToolBase):
 
             self.callback("gen_citations", {}, txt, elapsed_time=timer()-st)
 
+        def build_observation(tool_call_res: list[tuple]) -> str:
+            """
+            Build a Observation from tool call results.
+            No LLM involved.
+            """
+            if not tool_call_res:
+                return ""
+
+            lines = ["Observation:"]
+            for name, result in tool_call_res:
+                lines.append(f"[{name} result]")
+                lines.append(str(result))
+
+            return "\n".join(lines)
+
         def append_user_content(hist, content):
             if hist[-1]["role"] == "user":
                 hist[-1]["content"] += content
@@ -333,7 +371,7 @@ class Agent(LLM, ToolBase):
                 hist.append({"role": "user", "content": content})
 
         st = timer()
-        task_desc = await analyze_task_async(self.chat_mdl, prompt, user_request, tool_metas, user_defined_prompt)
+        task_desc = build_task_desc(prompt, user_request, user_defined_prompt)
         self.callback("analyze_task", {}, task_desc, elapsed_time=timer()-st)
         for _ in range(self._param.max_rounds + 1):
             if self.check_if_canceled("Agent streaming"):
@@ -364,7 +402,7 @@ class Agent(LLM, ToolBase):
 
                 results = await asyncio.gather(*tool_tasks) if tool_tasks else []
                 st = timer()
-                reflection = await reflect_async(self.chat_mdl, hist, results, user_defined_prompt)
+                reflection = build_observation(results)
                 append_user_content(hist, reflection)
                 self.callback("reflection", {}, str(reflection), elapsed_time=timer()-st)
 
@@ -393,11 +431,140 @@ Respond immediately with your final comprehensive answer.
         async for txt, tkcnt in complete():
             yield txt, tkcnt
 
+#     async def _react_with_tools_streamly_async(self, prompt, history: list[dict], use_tools, user_defined_prompt={}, schema_prompt: str = ""):
+#         token_count = 0
+#         tool_metas = self.tool_meta
+#         hist = deepcopy(history)
+#         last_calling = ""
+#         if len(hist) > 3:
+#             st = timer()
+#             user_request = await full_question(messages=history, chat_mdl=self.chat_mdl)
+#             self.callback("Multi-turn conversation optimization", {}, user_request, elapsed_time=timer()-st)
+#         else:
+#             user_request = history[-1]["content"]
+
+#         async def use_tool_async(name, args):
+#             nonlocal hist, use_tools, last_calling
+#             logging.info(f"{last_calling=} == {name=}")
+#             last_calling = name
+#             tool_response = await self.toolcall_session.tool_call_async(name, args)
+#             use_tools.append({
+#                 "name": name,
+#                 "arguments": args,
+#                 "results": tool_response
+#             })
+#             # self.callback("add_memory", {}, "...")
+#             #self.add_memory(hist[-2]["content"], hist[-1]["content"], name, args, str(tool_response), user_defined_prompt)
+
+#             return name, tool_response
+
+#         async def complete():
+#             nonlocal hist
+#             need2cite = self._param.cite and self._canvas.get_reference()["chunks"] and self._id.find("-->") < 0
+#             if schema_prompt:
+#                 need2cite = False
+#             cited = False
+#             if hist and hist[0]["role"] == "system":
+#                 if schema_prompt:
+#                     hist[0]["content"] += "\n" + schema_prompt
+#                 if need2cite and len(hist) < 7:
+#                     hist[0]["content"] += citation_prompt()
+#                     cited = True
+#             yield "", token_count
+
+#             _hist = hist
+#             if len(hist) > 12:
+#                 _hist = [hist[0], hist[1], *hist[-10:]]
+#             entire_txt = ""
+#             async for delta_ans in self._generate_streamly(_hist):
+#                 if not need2cite or cited:
+#                     yield delta_ans, 0
+#                 entire_txt += delta_ans
+#             if not need2cite or cited:
+#                 return
+
+#             st = timer()
+#             txt = ""
+#             async for delta_ans in self._gen_citations_async(entire_txt):
+#                 if self.check_if_canceled("Agent streaming"):
+#                     return
+#                 yield delta_ans, 0
+#                 txt += delta_ans
+
+#             self.callback("gen_citations", {}, txt, elapsed_time=timer()-st)
+
+#         def append_user_content(hist, content):
+#             if hist[-1]["role"] == "user":
+#                 hist[-1]["content"] += content
+#             else:
+#                 hist.append({"role": "user", "content": content})
+
+#         st = timer()
+#         task_desc = await analyze_task_async(self.chat_mdl, prompt, user_request, tool_metas, user_defined_prompt)
+#         self.callback("analyze_task", {}, task_desc, elapsed_time=timer()-st)
+#         for _ in range(self._param.max_rounds + 1):
+#             if self.check_if_canceled("Agent streaming"):
+#                 return
+#             response, tk = await next_step_async(self.chat_mdl, hist, tool_metas, task_desc, user_defined_prompt)
+#             # self.callback("next_step", {}, str(response)[:256]+"...")
+#             token_count += tk or 0
+#             hist.append({"role": "assistant", "content": response})
+#             try:
+#                 functions = json_repair.loads(re.sub(r"```.*", "", response))
+#                 if not isinstance(functions, list):
+#                     raise TypeError(f"List should be returned, but `{functions}`")
+#                 for f in functions:
+#                     if not isinstance(f, dict):
+#                         raise TypeError(f"An object type should be returned, but `{f}`")
+
+#                 tool_tasks = []
+#                 for func in functions:
+#                     name = func["name"]
+#                     args = func["arguments"]
+#                     if name == COMPLETE_TASK:
+#                         append_user_content(hist, f"Respond with a formal answer. FORGET(DO NOT mention) about `{COMPLETE_TASK}`. The language for the response MUST be as the same as the first user request.\n")
+#                         async for txt, tkcnt in complete():
+#                             yield txt, tkcnt
+#                         return
+
+#                     tool_tasks.append(asyncio.create_task(use_tool_async(name, args)))
+
+#                 results = await asyncio.gather(*tool_tasks) if tool_tasks else []
+#                 st = timer()
+#                 reflection = await reflect_async(self.chat_mdl, hist, results, user_defined_prompt)
+#                 append_user_content(hist, reflection)
+#                 self.callback("reflection", {}, str(reflection), elapsed_time=timer()-st)
+
+#             except Exception as e:
+#                 logging.exception(msg=f"Wrong JSON argument format in LLM ReAct response: {e}")
+#                 e = f"\nTool call error, please correct the input parameter of response format and call it again.\n *** Exception ***\n{e}"
+#                 append_user_content(hist, str(e))
+
+#         logging.warning( f"Exceed max rounds: {self._param.max_rounds}")
+#         final_instruction = f"""
+# {user_request}
+# IMPORTANT: You have reached the conversation limit. Based on ALL the information and research you have gathered so far, please provide a DIRECT and COMPREHENSIVE final answer to the original request.
+# Instructions:
+# 1. SYNTHESIZE all information collected during this conversation
+# 2. Provide a COMPLETE response using existing data - do not suggest additional research
+# 3. Structure your response as a FINAL DELIVERABLE, not a plan
+# 4. If information is incomplete, state what you found and provide the best analysis possible with available data
+# 5. DO NOT mention conversation limits or suggest further steps
+# 6. Focus on delivering VALUE with the information already gathered
+# Respond immediately with your final comprehensive answer.
+#         """
+#         if self.check_if_canceled("Agent final instruction"):
+#             return
+#         append_user_content(hist, final_instruction)
+
+#         async for txt, tkcnt in complete():
+#             yield txt, tkcnt
+
     async def _gen_citations_async(self, text):
         retrievals = self._canvas.get_reference()
         retrievals = {"chunks": list(retrievals["chunks"].values()), "doc_aggs": list(retrievals["doc_aggs"].values())}
         formated_refer = kb_prompt(retrievals, self.chat_mdl.max_length, True)
-        async for delta_ans in self._generate_streamly_async([{"role": "system", "content": citation_plus("\n\n".join(formated_refer))},
+        async for delta_ans in self._generate_streamly([{"role": "system", "content": citation_plus("\n\n".join(formated_refer))},
                                                   {"role": "user", "content": text}
                                                   ]):
             yield delta_ans

agent/component/base.py

@@ -24,11 +24,9 @@ import os
 import logging
 from typing import Any, List, Union
 import pandas as pd
-import trio
 from agent import settings
 from common.connection_utils import timeout
 
-
 _FEEDED_DEPRECATED_PARAMS = "_feeded_deprecated_params"
 _DEPRECATED_PARAMS = "_deprecated_params"
 _USER_FEEDED_PARAMS = "_user_feeded_params"
@@ -98,7 +96,7 @@ class ComponentParamBase(ABC):
         def _recursive_convert_obj_to_dict(obj):
             ret_dict = {}
             if isinstance(obj, dict):
-                for k,v in obj.items():
+                for k, v in obj.items():
                     if isinstance(v, dict) or (v and type(v).__name__ not in dir(builtins)):
                         ret_dict[k] = _recursive_convert_obj_to_dict(v)
                     else:
@@ -254,96 +252,65 @@ class ComponentParamBase(ABC):
                 self._validate_param(attr, validation_json)
 
     @staticmethod
-    def check_string(param, descr):
+    def check_string(param, description):
         if type(param).__name__ not in ["str"]:
-            raise ValueError(
-                descr + " {} not supported, should be string type".format(param)
-            )
+            raise ValueError(description + " {} not supported, should be string type".format(param))
 
     @staticmethod
-    def check_empty(param, descr):
+    def check_empty(param, description):
         if not param:
-            raise ValueError(
-                descr + " does not support empty value."
-            )
+            raise ValueError(description + " does not support empty value.")
 
     @staticmethod
-    def check_positive_integer(param, descr):
+    def check_positive_integer(param, description):
         if type(param).__name__ not in ["int", "long"] or param <= 0:
-            raise ValueError(
-                descr + " {} not supported, should be positive integer".format(param)
-            )
+            raise ValueError(description + " {} not supported, should be positive integer".format(param))
 
     @staticmethod
-    def check_positive_number(param, descr):
+    def check_positive_number(param, description):
         if type(param).__name__ not in ["float", "int", "long"] or param <= 0:
-            raise ValueError(
-                descr + " {} not supported, should be positive numeric".format(param)
-            )
+            raise ValueError(description + " {} not supported, should be positive numeric".format(param))
 
     @staticmethod
-    def check_nonnegative_number(param, descr):
+    def check_nonnegative_number(param, description):
         if type(param).__name__ not in ["float", "int", "long"] or param < 0:
-            raise ValueError(
-                descr
-                + " {} not supported, should be non-negative numeric".format(param)
-            )
+            raise ValueError(description + " {} not supported, should be non-negative numeric".format(param))
 
     @staticmethod
-    def check_decimal_float(param, descr):
+    def check_decimal_float(param, description):
         if type(param).__name__ not in ["float", "int"] or param < 0 or param > 1:
-            raise ValueError(
-                descr
-                + " {} not supported, should be a float number in range [0, 1]".format(
-                    param
-                )
-            )
+            raise ValueError(description + " {} not supported, should be a float number in range [0, 1]".format(param))
 
     @staticmethod
-    def check_boolean(param, descr):
+    def check_boolean(param, description):
         if type(param).__name__ != "bool":
-            raise ValueError(
-                descr + " {} not supported, should be bool type".format(param)
-            )
+            raise ValueError(description + " {} not supported, should be bool type".format(param))
 
     @staticmethod
-    def check_open_unit_interval(param, descr):
+    def check_open_unit_interval(param, description):
         if type(param).__name__ not in ["float"] or param <= 0 or param >= 1:
-            raise ValueError(
-                descr + " should be a numeric number between 0 and 1 exclusively"
-            )
+            raise ValueError(description + " should be a numeric number between 0 and 1 exclusively")
 
     @staticmethod
-    def check_valid_value(param, descr, valid_values):
+    def check_valid_value(param, description, valid_values):
         if param not in valid_values:
-            raise ValueError(
-                descr
-                + " {} is not supported, it should be in {}".format(param, valid_values)
-            )
+            raise ValueError(description + " {} is not supported, it should be in {}".format(param, valid_values))
 
     @staticmethod
-    def check_defined_type(param, descr, types):
+    def check_defined_type(param, description, types):
         if type(param).__name__ not in types:
-            raise ValueError(
-                descr + " {} not supported, should be one of {}".format(param, types)
-            )
+            raise ValueError(description + " {} not supported, should be one of {}".format(param, types))
 
     @staticmethod
-    def check_and_change_lower(param, valid_list, descr=""):
+    def check_and_change_lower(param, valid_list, description=""):
         if type(param).__name__ != "str":
-            raise ValueError(
-                descr
-                + " {} not supported, should be one of {}".format(param, valid_list)
-            )
+            raise ValueError(description + " {} not supported, should be one of {}".format(param, valid_list))
 
         lower_param = param.lower()
         if lower_param in valid_list:
             return lower_param
         else:
-            raise ValueError(
-                descr
-                + " {} not supported, should be one of {}".format(param, valid_list)
-            )
+            raise ValueError(description + " {} not supported, should be one of {}".format(param, valid_list))
 
     @staticmethod
     def _greater_equal_than(value, limit):
@@ -375,16 +342,16 @@ class ComponentParamBase(ABC):
     def _not_in(value, wrong_value_list):
         return value not in wrong_value_list
 
-    def _warn_deprecated_param(self, param_name, descr):
+    def _warn_deprecated_param(self, param_name, description):
         if self._deprecated_params_set.get(param_name):
             logging.warning(
-                f"{descr} {param_name} is deprecated and ignored in this version."
+                f"{description} {param_name} is deprecated and ignored in this version."
             )
 
-    def _warn_to_deprecate_param(self, param_name, descr, new_param):
+    def _warn_to_deprecate_param(self, param_name, description, new_param):
         if self._deprecated_params_set.get(param_name):
             logging.warning(
-                f"{descr} {param_name} will be deprecated in future release; "
+                f"{description} {param_name} will be deprecated in future release; "
                 f"please use {new_param} instead."
             )
             return True
@@ -393,8 +360,8 @@ class ComponentParamBase(ABC):
 
 class ComponentBase(ABC):
     component_name: str
-    thread_limiter = trio.CapacityLimiter(int(os.environ.get('MAX_CONCURRENT_CHATS', 10)))
-    variable_ref_patt = r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z0-9_.]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*"
+    thread_limiter = asyncio.Semaphore(int(os.environ.get("MAX_CONCURRENT_CHATS", 10)))
+    variable_ref_patt = r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z0-9_.-]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*"
 
     def __str__(self):
         """
@@ -408,7 +375,7 @@ class ComponentBase(ABC):
             "params": {}
         }}""".format(self.component_name,
                      self._param
-        )
+                     )
 
     def __init__(self, canvas, id, param: ComponentParamBase):
         from agent.canvas import Graph  # Local import to avoid cyclic dependency
@@ -474,14 +441,14 @@ class ComponentBase(ABC):
         self.set_output("_elapsed_time", time.perf_counter() - self.output("_created_time"))
         return self.output()
 
-    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
     def _invoke(self, **kwargs):
         raise NotImplementedError()
 
-    def output(self, var_nm: str=None) -> Union[dict[str, Any], Any]:
+    def output(self, var_nm: str = None) -> Union[dict[str, Any], Any]:
         if var_nm:
             return self._param.outputs.get(var_nm, {}).get("value", "")
-        return {k: o.get("value") for k,o in self._param.outputs.items()}
+        return {k: o.get("value") for k, o in self._param.outputs.items()}
 
     def set_output(self, key: str, value: Any):
         if key not in self._param.outputs:
@@ -492,18 +459,18 @@ class ComponentBase(ABC):
         return self._param.outputs.get("_ERROR", {}).get("value")
 
     def reset(self, only_output=False):
-        outputs: dict = self._param.outputs # for better performance
+        outputs: dict = self._param.outputs  # for better performance
         for k in outputs.keys():
             outputs[k]["value"] = None
         if only_output:
             return
 
-        inputs: dict = self._param.inputs # for better performance
+        inputs: dict = self._param.inputs  # for better performance
         for k in inputs.keys():
             inputs[k]["value"] = None
         self._param.debug_inputs = {}
 
-    def get_input(self, key: str=None) -> Union[Any, dict[str, Any]]:
+    def get_input(self, key: str = None) -> Union[Any, dict[str, Any]]:
         if key:
             return self._param.inputs.get(key, {}).get("value")
 
@@ -527,13 +494,13 @@ class ComponentBase(ABC):
 
     def get_input_elements_from_text(self, txt: str) -> dict[str, dict[str, str]]:
         res = {}
-        for r in re.finditer(self.variable_ref_patt, txt, flags=re.IGNORECASE|re.DOTALL):
+        for r in re.finditer(self.variable_ref_patt, txt, flags=re.IGNORECASE | re.DOTALL):
             exp = r.group(1)
-            cpn_id, var_nm = exp.split("@") if exp.find("@")>0 else ("", exp)
+            cpn_id, var_nm = exp.split("@") if exp.find("@") > 0 else ("", exp)
             res[exp] = {
-                "name": (self._canvas.get_component_name(cpn_id) +f"@{var_nm}") if cpn_id else exp,
+                "name": (self._canvas.get_component_name(cpn_id) + f"@{var_nm}") if cpn_id else exp,
                 "value": self._canvas.get_variable_value(exp),
-                "_retrival": self._canvas.get_variable_value(f"{cpn_id}@_references") if cpn_id else None,
+                "_retrieval": self._canvas.get_variable_value(f"{cpn_id}@_references") if cpn_id else None,
                 "_cpn_id": cpn_id
             }
         return res
@@ -584,6 +551,7 @@ class ComponentBase(ABC):
         for n, v in kv.items():
             def repl(_match, val=v):
                 return str(val) if val is not None else ""
+
             content = re.sub(
                 r"\{%s\}" % re.escape(n),
                 repl,

agent/component/begin.py

@@ -28,7 +28,7 @@ class BeginParam(UserFillUpParam):
         self.prologue = "Hi! I'm your smart assistant. What can I do for you?"
 
     def check(self):
-        self.check_valid_value(self.mode, "The 'mode' should be either `conversational` or `task`", ["conversational", "task"])
+        self.check_valid_value(self.mode, "The 'mode' should be either `conversational` or `task`", ["conversational", "task","Webhook"])
 
     def get_input_form(self) -> dict[str, dict]:
         return getattr(self, "inputs")

agent/component/categorize.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import asyncio
 import logging
 import os
 import re
@@ -97,7 +98,7 @@ class Categorize(LLM, ABC):
     component_name = "Categorize"
 
     @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
-    def _invoke(self, **kwargs):
+    async def _invoke_async(self, **kwargs):
         if self.check_if_canceled("Categorize processing"):
             return
 
@@ -121,7 +122,7 @@ class Categorize(LLM, ABC):
         if self.check_if_canceled("Categorize processing"):
             return
 
-        ans = chat_mdl.chat(self._param.sys_prompt, [{"role": "user", "content": user_prompt}], self._param.gen_conf())
+        ans = await chat_mdl.async_chat(self._param.sys_prompt, [{"role": "user", "content": user_prompt}], self._param.gen_conf())
         logging.info(f"input: {user_prompt}, answer: {str(ans)}")
         if ERROR_PREFIX in ans:
             raise Exception(ans)
@@ -144,5 +145,9 @@ class Categorize(LLM, ABC):
         self.set_output("category_name", max_category)
         self.set_output("_next", cpn_ids)
 
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
+    def _invoke(self, **kwargs):
+        return asyncio.run(self._invoke_async(**kwargs))
+
     def thoughts(self) -> str:
         return "Which should it falls into {}? ...".format(",".join([f"`{c}`" for c, _ in self._param.category_description.items()]))

agent/component/excel_processor.py

@@ -0,0 +1,401 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+"""
+ExcelProcessor Component
+
+A component for reading, processing, and generating Excel files in RAGFlow agents.
+Supports multiple Excel file inputs, data transformation, and Excel output generation.
+"""
+
+import logging
+import os
+from abc import ABC
+from io import BytesIO
+
+import pandas as pd
+
+from agent.component.base import ComponentBase, ComponentParamBase
+from api.db.services.file_service import FileService
+from api.utils.api_utils import timeout
+from common import settings
+from common.misc_utils import get_uuid
+
+
+class ExcelProcessorParam(ComponentParamBase):
+    """
+    Define the ExcelProcessor component parameters.
+    """
+    def __init__(self):
+        super().__init__()
+        # Input configuration
+        self.input_files = []  # Variable references to uploaded files
+        self.operation = "read"  # read, merge, transform, output
+        
+        # Processing options
+        self.sheet_selection = "all"  # all, first, or comma-separated sheet names
+        self.merge_strategy = "concat"  # concat, join
+        self.join_on = ""  # Column name for join operations
+        
+        # Transform options (for LLM-guided transformations)
+        self.transform_instructions = ""
+        self.transform_data = ""  # Variable reference to transformation data
+        
+        # Output options
+        self.output_format = "xlsx"  # xlsx, csv
+        self.output_filename = "output"
+        
+        # Component outputs
+        self.outputs = {
+            "data": {
+                "type": "object",
+                "value": {}
+            },
+            "summary": {
+                "type": "str",
+                "value": ""
+            },
+            "markdown": {
+                "type": "str",
+                "value": ""
+            }
+        }
+    
+    def check(self):
+        self.check_valid_value(
+            self.operation, 
+            "[ExcelProcessor] Operation", 
+            ["read", "merge", "transform", "output"]
+        )
+        self.check_valid_value(
+            self.output_format,
+            "[ExcelProcessor] Output format",
+            ["xlsx", "csv"]
+        )
+        return True
+
+
+class ExcelProcessor(ComponentBase, ABC):
+    """
+    Excel processing component for RAGFlow agents.
+    
+    Operations:
+    - read: Parse Excel files into structured data
+    - merge: Combine multiple Excel files
+    - transform: Apply data transformations based on instructions
+    - output: Generate Excel file output
+    """
+    component_name = "ExcelProcessor"
+
+    def get_input_form(self) -> dict[str, dict]:
+        """Define input form for the component."""
+        res = {}
+        for ref in (self._param.input_files or []):
+            for k, o in self.get_input_elements_from_text(ref).items():
+                res[k] = {"name": o.get("name", ""), "type": "file"}
+        if self._param.transform_data:
+            for k, o in self.get_input_elements_from_text(self._param.transform_data).items():
+                res[k] = {"name": o.get("name", ""), "type": "object"}
+        return res
+
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
+    def _invoke(self, **kwargs):
+        if self.check_if_canceled("ExcelProcessor processing"):
+            return
+
+        operation = self._param.operation.lower()
+        
+        if operation == "read":
+            self._read_excels()
+        elif operation == "merge":
+            self._merge_excels()
+        elif operation == "transform":
+            self._transform_data()
+        elif operation == "output":
+            self._output_excel()
+        else:
+            self.set_output("summary", f"Unknown operation: {operation}")
+
+    def _get_file_content(self, file_ref: str) -> tuple[bytes, str]:
+        """
+        Get file content from a variable reference.
+        Returns (content_bytes, filename).
+        """
+        value = self._canvas.get_variable_value(file_ref)
+        if value is None:
+            return None, None
+            
+        # Handle different value formats
+        if isinstance(value, dict):
+            # File reference from Begin/UserFillUp component
+            file_id = value.get("id") or value.get("file_id")
+            created_by = value.get("created_by") or self._canvas.get_tenant_id()
+            filename = value.get("name") or value.get("filename", "unknown.xlsx")
+            if file_id:
+                content = FileService.get_blob(created_by, file_id)
+                return content, filename
+        elif isinstance(value, list) and len(value) > 0:
+            # List of file references - return first
+            return self._get_file_content_from_list(value[0])
+        elif isinstance(value, str):
+            # Could be base64 encoded or a path
+            if value.startswith("data:"):
+                import base64
+                # Extract base64 content
+                _, encoded = value.split(",", 1)
+                return base64.b64decode(encoded), "uploaded.xlsx"
+                
+        return None, None
+    
+    def _get_file_content_from_list(self, item) -> tuple[bytes, str]:
+        """Extract file content from a list item."""
+        if isinstance(item, dict):
+            return self._get_file_content(item)
+        return None, None
+
+    def _parse_excel_to_dataframes(self, content: bytes, filename: str) -> dict[str, pd.DataFrame]:
+        """Parse Excel content into a dictionary of DataFrames (one per sheet)."""
+        try:
+            excel_file = BytesIO(content)
+            
+            if filename.lower().endswith(".csv"):
+                df = pd.read_csv(excel_file)
+                return {"Sheet1": df}
+            else:
+                # Read all sheets
+                xlsx = pd.ExcelFile(excel_file, engine='openpyxl')
+                sheet_selection = self._param.sheet_selection
+                
+                if sheet_selection == "all":
+                    sheets_to_read = xlsx.sheet_names
+                elif sheet_selection == "first":
+                    sheets_to_read = [xlsx.sheet_names[0]] if xlsx.sheet_names else []
+                else:
+                    # Comma-separated sheet names
+                    requested = [s.strip() for s in sheet_selection.split(",")]
+                    sheets_to_read = [s for s in requested if s in xlsx.sheet_names]
+                
+                dfs = {}
+                for sheet in sheets_to_read:
+                    dfs[sheet] = pd.read_excel(xlsx, sheet_name=sheet)
+                return dfs
+                
+        except Exception as e:
+            logging.error(f"Error parsing Excel file {filename}: {e}")
+            return {}
+
+    def _read_excels(self):
+        """Read and parse Excel files into structured data."""
+        all_data = {}
+        summaries = []
+        markdown_parts = []
+        
+        for file_ref in (self._param.input_files or []):
+            if self.check_if_canceled("ExcelProcessor reading"):
+                return
+                
+            # Get variable value
+            value = self._canvas.get_variable_value(file_ref)
+            self.set_input_value(file_ref, str(value)[:200] if value else "")
+            
+            if value is None:
+                continue
+            
+            # Handle file content
+            content, filename = self._get_file_content(file_ref)
+            if content is None:
+                continue
+                
+            # Parse Excel
+            dfs = self._parse_excel_to_dataframes(content, filename)
+            
+            for sheet_name, df in dfs.items():
+                key = f"{filename}_{sheet_name}" if len(dfs) > 1 else filename
+                all_data[key] = df.to_dict(orient="records")
+                
+                # Build summary
+                summaries.append(f"**{key}**: {len(df)} rows, {len(df.columns)} columns ({', '.join(df.columns.tolist()[:5])}{'...' if len(df.columns) > 5 else ''})")
+                
+                # Build markdown table
+                markdown_parts.append(f"### {key}\n\n{df.head(10).to_markdown(index=False)}\n")
+        
+        # Set outputs
+        self.set_output("data", all_data)
+        self.set_output("summary", "\n".join(summaries) if summaries else "No Excel files found")
+        self.set_output("markdown", "\n\n".join(markdown_parts) if markdown_parts else "No data")
+
+    def _merge_excels(self):
+        """Merge multiple Excel files/sheets into one."""
+        all_dfs = []
+        
+        for file_ref in (self._param.input_files or []):
+            if self.check_if_canceled("ExcelProcessor merging"):
+                return
+                
+            value = self._canvas.get_variable_value(file_ref)
+            self.set_input_value(file_ref, str(value)[:200] if value else "")
+            
+            if value is None:
+                continue
+                
+            content, filename = self._get_file_content(file_ref)
+            if content is None:
+                continue
+                
+            dfs = self._parse_excel_to_dataframes(content, filename)
+            all_dfs.extend(dfs.values())
+        
+        if not all_dfs:
+            self.set_output("data", {})
+            self.set_output("summary", "No data to merge")
+            return
+        
+        # Merge strategy
+        if self._param.merge_strategy == "concat":
+            merged_df = pd.concat(all_dfs, ignore_index=True)
+        elif self._param.merge_strategy == "join" and self._param.join_on:
+            # Join on specified column
+            merged_df = all_dfs[0]
+            for df in all_dfs[1:]:
+                merged_df = merged_df.merge(df, on=self._param.join_on, how="outer")
+        else:
+            merged_df = pd.concat(all_dfs, ignore_index=True)
+        
+        self.set_output("data", {"merged": merged_df.to_dict(orient="records")})
+        self.set_output("summary", f"Merged {len(all_dfs)} sources into {len(merged_df)} rows, {len(merged_df.columns)} columns")
+        self.set_output("markdown", merged_df.head(20).to_markdown(index=False))
+
+    def _transform_data(self):
+        """Apply transformations to data based on instructions or input data."""
+        # Get the data to transform
+        transform_ref = self._param.transform_data
+        if not transform_ref:
+            self.set_output("summary", "No transform data reference provided")
+            return
+            
+        data = self._canvas.get_variable_value(transform_ref)
+        self.set_input_value(transform_ref, str(data)[:300] if data else "")
+        
+        if data is None:
+            self.set_output("summary", "Transform data is empty")
+            return
+        
+        # Convert to DataFrame
+        if isinstance(data, dict):
+            # Could be {"sheet": [rows]} format
+            if all(isinstance(v, list) for v in data.values()):
+                # Multiple sheets
+                all_markdown = []
+                for sheet_name, rows in data.items():
+                    df = pd.DataFrame(rows)
+                    all_markdown.append(f"### {sheet_name}\n\n{df.to_markdown(index=False)}")
+                self.set_output("data", data)
+                self.set_output("markdown", "\n\n".join(all_markdown))
+            else:
+                df = pd.DataFrame([data])
+                self.set_output("data", df.to_dict(orient="records"))
+                self.set_output("markdown", df.to_markdown(index=False))
+        elif isinstance(data, list):
+            df = pd.DataFrame(data)
+            self.set_output("data", df.to_dict(orient="records"))
+            self.set_output("markdown", df.to_markdown(index=False))
+        else:
+            self.set_output("data", {"raw": str(data)})
+            self.set_output("markdown", str(data))
+        
+        self.set_output("summary", "Transformed data ready for processing")
+
+    def _output_excel(self):
+        """Generate Excel file output from data."""
+        # Get data from transform_data reference
+        transform_ref = self._param.transform_data
+        if not transform_ref:
+            self.set_output("summary", "No data reference for output")
+            return
+            
+        data = self._canvas.get_variable_value(transform_ref)
+        self.set_input_value(transform_ref, str(data)[:300] if data else "")
+        
+        if data is None:
+            self.set_output("summary", "No data to output")
+            return
+        
+        try:
+            # Prepare DataFrames
+            if isinstance(data, dict):
+                if all(isinstance(v, list) for v in data.values()):
+                    # Multi-sheet format
+                    dfs = {k: pd.DataFrame(v) for k, v in data.items()}
+                else:
+                    dfs = {"Sheet1": pd.DataFrame([data])}
+            elif isinstance(data, list):
+                dfs = {"Sheet1": pd.DataFrame(data)}
+            else:
+                self.set_output("summary", "Invalid data format for Excel output")
+                return
+            
+            # Generate output
+            doc_id = get_uuid()
+            
+            if self._param.output_format == "csv":
+                # For CSV, only output first sheet
+                first_df = list(dfs.values())[0]
+                binary_content = first_df.to_csv(index=False).encode("utf-8")
+                filename = f"{self._param.output_filename}.csv"
+            else:
+                # Excel output
+                excel_io = BytesIO()
+                with pd.ExcelWriter(excel_io, engine='openpyxl') as writer:
+                    for sheet_name, df in dfs.items():
+                        # Sanitize sheet name (max 31 chars, no special chars)
+                        safe_name = sheet_name[:31].replace("/", "_").replace("\\", "_")
+                        df.to_excel(writer, sheet_name=safe_name, index=False)
+                excel_io.seek(0)
+                binary_content = excel_io.read()
+                filename = f"{self._param.output_filename}.xlsx"
+            
+            # Store file
+            settings.STORAGE_IMPL.put(self._canvas._tenant_id, doc_id, binary_content)
+            
+            # Set attachment output
+            self.set_output("attachment", {
+                "doc_id": doc_id,
+                "format": self._param.output_format,
+                "file_name": filename
+            })
+            
+            total_rows = sum(len(df) for df in dfs.values())
+            self.set_output("summary", f"Generated {filename} with {len(dfs)} sheet(s), {total_rows} total rows")
+            self.set_output("data", {k: v.to_dict(orient="records") for k, v in dfs.items()})
+            
+            logging.info(f"ExcelProcessor: Generated {filename} as {doc_id}")
+            
+        except Exception as e:
+            logging.error(f"ExcelProcessor output error: {e}")
+            self.set_output("summary", f"Error generating output: {str(e)}")
+
+    def thoughts(self) -> str:
+        """Return component thoughts for UI display."""
+        op = self._param.operation
+        if op == "read":
+            return "Reading Excel files..."
+        elif op == "merge":
+            return "Merging Excel data..."
+        elif op == "transform":
+            return "Transforming data..."
+        elif op == "output":
+            return "Generating Excel output..."
+        return "Processing Excel..."

agent/component/fillup.py

@@ -18,6 +18,7 @@ import re
 from functools import partial
 
 from agent.component.base import ComponentParamBase, ComponentBase
+from api.db.services.file_service import FileService
 
 
 class UserFillUpParam(ComponentParamBase):
@@ -63,6 +64,13 @@ class UserFillUp(ComponentBase):
         for k, v in kwargs.get("inputs", {}).items():
             if self.check_if_canceled("UserFillUp processing"):
                 return
+            if isinstance(v, dict) and v.get("type", "").lower().find("file") >=0:
+                if v.get("optional") and v.get("value", None) is None:
+                    v = None
+                else:
+                    v = FileService.get_files([v["value"]])
+            else:
+                v = v.get("value")
             self.set_output(k, v)
 
     def thoughts(self) -> str:

agent/component/llm.py

@@ -18,9 +18,8 @@ import json
 import logging
 import os
 import re
-import threading
 from copy import deepcopy
-from typing import Any, Generator, AsyncGenerator
+from typing import Any, AsyncGenerator
 import json_repair
 from functools import partial
 from common.constants import LLMType
@@ -57,7 +56,6 @@ class LLMParam(ComponentParamBase):
         self.check_nonnegative_number(int(self.max_tokens), "[Agent] Max tokens")
         self.check_decimal_float(float(self.top_p), "[Agent] Top P")
         self.check_empty(self.llm_id, "[Agent] LLM")
-        self.check_empty(self.sys_prompt, "[Agent] System prompt")
         self.check_empty(self.prompts, "[Agent] User prompt")
 
     def gen_conf(self):
@@ -168,53 +166,12 @@ class LLM(ComponentBase):
             sys_prompt = re.sub(rf"<{tag}>(.*?)</{tag}>", "", sys_prompt, flags=re.DOTALL|re.IGNORECASE)
         return pts, sys_prompt
 
-    def _generate(self, msg:list[dict], **kwargs) -> str:
-        if not self.imgs:
-            return self.chat_mdl.chat(msg[0]["content"], msg[1:], self._param.gen_conf(), **kwargs)
-        return self.chat_mdl.chat(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)
-
     async def _generate_async(self, msg: list[dict], **kwargs) -> str:
-        if not self.imgs and hasattr(self.chat_mdl, "async_chat"):
-            return await self.chat_mdl.async_chat(msg[0]["content"], msg[1:], self._param.gen_conf(), **kwargs)
-        if self.imgs and hasattr(self.chat_mdl, "async_chat"):
-            return await self.chat_mdl.async_chat(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)
-        return await asyncio.to_thread(self._generate, msg, **kwargs)
-
-    def _generate_streamly(self, msg:list[dict], **kwargs) -> Generator[str, None, None]:
-        ans = ""
-        last_idx = 0
-        endswith_think = False
-        def delta(txt):
-            nonlocal ans, last_idx, endswith_think
-            delta_ans = txt[last_idx:]
-            ans = txt
-
-            if delta_ans.find("<think>") == 0:
-                last_idx += len("<think>")
-                return "<think>"
-            elif delta_ans.find("<think>") > 0:
-                delta_ans = txt[last_idx:last_idx+delta_ans.find("<think>")]
-                last_idx += delta_ans.find("<think>")
-                return delta_ans
-            elif delta_ans.endswith("</think>"):
-                endswith_think = True
-            elif endswith_think:
-                endswith_think = False
-                return "</think>"
-
-            last_idx = len(ans)
-            if ans.endswith("</think>"):
-                last_idx -= len("</think>")
-            return re.sub(r"(<think>|</think>)", "", delta_ans)
-
         if not self.imgs:
-            for txt in self.chat_mdl.chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), **kwargs):
-                yield delta(txt)
-        else:
-            for txt in self.chat_mdl.chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs):
-                yield delta(txt)
+            return await self.chat_mdl.async_chat(msg[0]["content"], msg[1:], self._param.gen_conf(), **kwargs)
+        return await self.chat_mdl.async_chat(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)
 
-    async def _generate_streamly_async(self, msg: list[dict], **kwargs) -> AsyncGenerator[str, None]:
+    async def _generate_streamly(self, msg: list[dict], **kwargs) -> AsyncGenerator[str, None]:
         async def delta_wrapper(txt_iter):
             ans = ""
             last_idx = 0
@@ -246,36 +203,13 @@ class LLM(ComponentBase):
             async for t in txt_iter:
                 yield delta(t)
 
-        if not self.imgs and hasattr(self.chat_mdl, "async_chat_streamly"):
+        if not self.imgs:
             async for t in delta_wrapper(self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), **kwargs)):
                 yield t
             return
-        if self.imgs and hasattr(self.chat_mdl, "async_chat_streamly"):
-            async for t in delta_wrapper(self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)):
-                yield t
-            return
 
-        # fallback
-        loop = asyncio.get_running_loop()
-        queue: asyncio.Queue = asyncio.Queue()
-
-        def worker():
-            try:
-                for item in self._generate_streamly(msg, **kwargs):
-                    loop.call_soon_threadsafe(queue.put_nowait, item)
-            except Exception as e:
-                loop.call_soon_threadsafe(queue.put_nowait, e)
-            finally:
-                loop.call_soon_threadsafe(queue.put_nowait, StopAsyncIteration)
-
-        threading.Thread(target=worker, daemon=True).start()
-        while True:
-            item = await queue.get()
-            if item is StopAsyncIteration:
-                break
-            if isinstance(item, Exception):
-                raise item
-            yield item
+        async for t in delta_wrapper(self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)):
+            yield t
 
     async def _stream_output_async(self, prompt, msg):
         _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97))
@@ -407,8 +341,8 @@ class LLM(ComponentBase):
     def _invoke(self, **kwargs):
         return asyncio.run(self._invoke_async(**kwargs))
 
-    def add_memory(self, user:str, assist:str, func_name: str, params: dict, results: str, user_defined_prompt:dict={}):
-        summ = tool_call_summary(self.chat_mdl, func_name, params, results, user_defined_prompt)
+    async def add_memory(self, user:str, assist:str, func_name: str, params: dict, results: str, user_defined_prompt:dict={}):
+        summ = await tool_call_summary(self.chat_mdl, func_name, params, results, user_defined_prompt)
         logging.info(f"[MEMORY]: {summ}")
         self._canvas.add_memory(user, assist, summ)
 

agent/component/loopitem.py

@@ -113,6 +113,10 @@ class LoopItem(ComponentBase, ABC):
                 return len(var) == 0
             elif operator == "not empty":
                 return len(var) > 0
+        elif var is None:
+            if operator == "empty":
+                return True
+            return False
 
         raise Exception(f"Invalid operator: {operator}")
 

agent/component/message.py

@@ -14,6 +14,8 @@
 #  limitations under the License.
 #
 import asyncio
+import nest_asyncio
+nest_asyncio.apply()
 import inspect
 import json
 import os
@@ -31,6 +33,8 @@ from common.connection_utils import timeout
 from common.misc_utils import get_uuid
 from common import settings
 
+from api.db.joint_services.memory_message_service import save_to_memory
+
 
 class MessageParam(ComponentParamBase):
     """
@@ -164,6 +168,7 @@ class Message(ComponentBase):
 
         self.set_output("content", all_content)
         self._convert_content(all_content)
+        await self._save_to_memory(all_content)
 
     def _is_jinjia2(self, content:str) -> bool:
         patt = [
@@ -196,10 +201,53 @@ class Message(ComponentBase):
 
         self.set_output("content", content)
         self._convert_content(content)
+        self._save_to_memory(content)
 
     def thoughts(self) -> str:
         return ""
 
+    def _parse_markdown_table_lines(self, table_lines: list):
+        """
+        Parse a list of Markdown table lines into a pandas DataFrame.
+        
+        Args:
+            table_lines: List of strings, each representing a row in the Markdown table
+                        (excluding separator lines like |---|---|)
+        
+        Returns:
+            pandas DataFrame with the table data, or None if parsing fails
+        """
+        import pandas as pd
+        
+        if not table_lines:
+            return None
+        
+        rows = []
+        headers = None
+        
+        for line in table_lines:
+            # Split by | and clean up
+            cells = [cell.strip() for cell in line.split('|')]
+            # Remove empty first and last elements from split (caused by leading/trailing |)
+            cells = [c for c in cells if c]
+            
+            if headers is None:
+                headers = cells
+            else:
+                rows.append(cells)
+        
+        if headers and rows:
+            # Ensure all rows have same number of columns as headers
+            normalized_rows = []
+            for row in rows:
+                while len(row) < len(headers):
+                    row.append('')
+                normalized_rows.append(row[:len(headers)])
+            
+            return pd.DataFrame(normalized_rows, columns=headers)
+        
+        return None
+
     def _convert_content(self, content):
         if not self._param.output_format:
             return
@@ -207,7 +255,7 @@ class Message(ComponentBase):
         import pypandoc
         doc_id = get_uuid()
 
-        if self._param.output_format.lower() not in {"markdown", "html", "pdf", "docx"}:
+        if self._param.output_format.lower() not in {"markdown", "html", "pdf", "docx", "xlsx"}:
             self._param.output_format = "markdown"
 
         try:
@@ -227,6 +275,119 @@ class Message(ComponentBase):
 
                 binary_content = converted.encode("utf-8")
 
+            elif self._param.output_format == "xlsx":
+                import pandas as pd
+                from io import BytesIO
+
+                # Debug: log the content being parsed
+                logging.info(f"XLSX Parser: Content length={len(content) if content else 0}, first 500 chars: {content[:500] if content else 'None'}")
+                
+                # Try to parse ALL Markdown tables from the content
+                # Each table will be written to a separate sheet
+                tables = []  # List of (sheet_name, dataframe)
+                
+                if isinstance(content, str):
+                    lines = content.strip().split('\n')
+                    logging.info(f"XLSX Parser: Total lines={len(lines)}, lines starting with '|': {sum(1 for line in lines if line.strip().startswith('|'))}")
+                    current_table_lines = []
+                    current_table_title = None
+                    pending_title = None
+                    in_table = False
+                    table_count = 0
+                    
+                    for i, line in enumerate(lines):
+                        stripped = line.strip()
+                        
+                        # Check for potential table title (lines before a table)
+                        # Look for patterns like "Table 1:", "## Table", or markdown headers
+                        if not in_table and stripped and not stripped.startswith('|'):
+                            # Check if this could be a table title
+                            lower_stripped = stripped.lower()
+                            if (lower_stripped.startswith('table') or 
+                                stripped.startswith('#') or
+                                ':' in stripped):
+                                pending_title = stripped.lstrip('#').strip()
+                        
+                        if stripped.startswith('|') and '|' in stripped[1:]:
+                            # Check if this is a separator line (|---|---|)
+                            cleaned = stripped.replace(' ', '').replace('|', '').replace('-', '').replace(':', '')
+                            if cleaned == '':
+                                continue  # Skip separator line
+                            
+                            if not in_table:
+                                # Starting a new table
+                                in_table = True
+                                current_table_lines = []
+                                current_table_title = pending_title
+                                pending_title = None
+                            
+                            current_table_lines.append(stripped)
+                        
+                        elif in_table and not stripped.startswith('|'):
+                            # End of current table - save it
+                            if current_table_lines:
+                                df = self._parse_markdown_table_lines(current_table_lines)
+                                if df is not None and not df.empty:
+                                    table_count += 1
+                                    # Generate sheet name
+                                    if current_table_title:
+                                        # Clean and truncate title for sheet name
+                                        sheet_name = current_table_title[:31]
+                                        sheet_name = sheet_name.replace('/', '_').replace('\\', '_').replace('*', '').replace('?', '').replace('[', '').replace(']', '').replace(':', '')
+                                    else:
+                                        sheet_name = f"Table_{table_count}"
+                                    tables.append((sheet_name, df))
+                            
+                            # Reset for next table
+                            in_table = False
+                            current_table_lines = []
+                            current_table_title = None
+                            
+                            # Check if this line could be a title for the next table
+                            if stripped:
+                                lower_stripped = stripped.lower()
+                                if (lower_stripped.startswith('table') or 
+                                    stripped.startswith('#') or
+                                    ':' in stripped):
+                                    pending_title = stripped.lstrip('#').strip()
+                    
+                    # Don't forget the last table if content ends with a table
+                    if in_table and current_table_lines:
+                        df = self._parse_markdown_table_lines(current_table_lines)
+                        if df is not None and not df.empty:
+                            table_count += 1
+                            if current_table_title:
+                                sheet_name = current_table_title[:31]
+                                sheet_name = sheet_name.replace('/', '_').replace('\\', '_').replace('*', '').replace('?', '').replace('[', '').replace(']', '').replace(':', '')
+                            else:
+                                sheet_name = f"Table_{table_count}"
+                            tables.append((sheet_name, df))
+                
+                # Fallback: if no tables found, create single sheet with content
+                if not tables:
+                    df = pd.DataFrame({"Content": [content if content else ""]})
+                    tables = [("Data", df)]
+
+                # Write all tables to Excel, each in a separate sheet
+                excel_io = BytesIO()
+                with pd.ExcelWriter(excel_io, engine='openpyxl') as writer:
+                    used_names = set()
+                    for sheet_name, df in tables:
+                        # Ensure unique sheet names
+                        original_name = sheet_name
+                        counter = 1
+                        while sheet_name in used_names:
+                            suffix = f"_{counter}"
+                            sheet_name = original_name[:31-len(suffix)] + suffix
+                            counter += 1
+                        used_names.add(sheet_name)
+                        df.to_excel(writer, sheet_name=sheet_name, index=False)
+                
+                excel_io.seek(0)
+                binary_content = excel_io.read()
+                
+                logging.info(f"Generated Excel with {len(tables)} sheet(s): {[t[0] for t in tables]}")
+
             else:  # pdf, docx
                 with tempfile.NamedTemporaryFile(suffix=f".{self._param.output_format}", delete=False) as tmp:
                     tmp_name = tmp.name
@@ -264,3 +425,29 @@ class Message(ComponentBase):
 
         except Exception as e:
             logging.error(f"Error converting content to {self._param.output_format}: {e}")
+
+    async def _save_to_memory(self, content):
+        if not hasattr(self._param, "memory_ids") or not self._param.memory_ids:
+            return True, "No memory selected."
+
+        message_dict = {
+            "user_id": self._canvas._tenant_id,
+            "agent_id": self._canvas._id,
+            "session_id": self._canvas.task_id,
+            "user_input": self._canvas.get_sys_query(),
+            "agent_response": content
+        }
+        res = []
+        for memory_id in self._param.memory_ids:
+            success, msg = await save_to_memory(memory_id, message_dict)
+            res.append({
+                "memory_id": memory_id,
+                "success": success,
+                "msg": msg
+            })
+        if all([r["success"] for r in res]):
+            return True, "Successfully added to memories."
+
+        error_text = "Some messages failed to add. " + " ".join([f"Add to memory {r['memory_id']} failed, detail: {r['msg']}" for r in res if not r["success"]])
+        logging.error(error_text)
+        return False, error_text

agent/templates/advanced_ingestion_pipeline.json

@@ -193,7 +193,7 @@
                         "presence_penalty": 0.4,
                         "prompts": [
                             {
-                                "content": "Text Content:\n{Splitter:KindDingosJam@chunks}\n",
+                                "content": "Text Content:\n{Splitter:NineTiesSin@chunks}\n",
                                 "role": "user"
                             }
                         ],
@@ -226,7 +226,7 @@
                         "presence_penalty": 0.4,
                         "prompts": [
                             {
-                                "content": "Text Content:\n\n{Splitter:KindDingosJam@chunks}\n",
+                                "content": "Text Content:\n\n{Splitter:TastyPointsLay@chunks}\n",
                                 "role": "user"
                             }
                         ],
@@ -259,7 +259,7 @@
                         "presence_penalty": 0.4,
                         "prompts": [
                             {
-                                "content": "Content: \n\n{Splitter:KindDingosJam@chunks}",
+                                "content": "Content: \n\n{Splitter:CuteBusesBet@chunks}",
                                 "role": "user"
                             }
                         ],
@@ -485,7 +485,7 @@
                             "outputs": {},
                             "presencePenaltyEnabled": false,
                             "presence_penalty": 0.4,
-                            "prompts": "Text Content:\n{Splitter:KindDingosJam@chunks}\n",
+                            "prompts": "Text Content:\n{Splitter:NineTiesSin@chunks}\n",
                             "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.",
                             "temperature": 0.1,
                             "temperatureEnabled": false,
@@ -522,7 +522,7 @@
                             "outputs": {},
                             "presencePenaltyEnabled": false,
                             "presence_penalty": 0.4,
-                            "prompts": "Text Content:\n\n{Splitter:KindDingosJam@chunks}\n",
+                            "prompts": "Text Content:\n\n{Splitter:TastyPointsLay@chunks}\n",
                             "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.",
                             "temperature": 0.1,
                             "temperatureEnabled": false,
@@ -559,7 +559,7 @@
                             "outputs": {},
                             "presencePenaltyEnabled": false,
                             "presence_penalty": 0.4,
-                            "prompts": "Content: \n\n{Splitter:KindDingosJam@chunks}",
+                            "prompts": "Content: \n\n{Splitter:BlueResultsWink@chunks}",
                             "sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.",
                             "temperature": 0.1,
                             "temperatureEnabled": false,

agent/templates/sql_assistant.json

@@ -578,7 +578,7 @@
                     {
                         "data": {
                             "form": {
-                                "text": "Searches for relevant database creation statements.\n\nIt should label with a knowledgebase to which the schema is dumped in. You could use \" General \" as parsing method, \" 2 \" as chunk size and \" ; \"  as delimiter."
+                                "text": "Searches for relevant database creation statements.\n\nIt should label with a dataset to which the schema is dumped in. You could use \" General \" as parsing method, \" 2 \" as chunk size and \" ; \"  as delimiter."
                             },
                             "label": "Note",
                             "name": "Note Schema"

agent/test/dsl_examples/categorize_and_agent_with_tavily.json

@@ -75,7 +75,7 @@
   },
   "history": [],
   "path": [],
-  "retrival": {"chunks": [], "doc_aggs": []},
+  "retrieval": {"chunks": [], "doc_aggs": []},
   "globals": {
     "sys.query": "",
     "sys.user_id": "",

agent/test/dsl_examples/iteration.json

@@ -82,7 +82,7 @@
   },
   "history": [],
   "path": [],
-  "retrival": {"chunks": [], "doc_aggs": []},
+  "retrieval": {"chunks": [], "doc_aggs": []},
   "globals": {
     "sys.query": "",
     "sys.user_id": "",

agent/test/dsl_examples/retrieval_and_generate.json

@@ -31,7 +31,7 @@
                     "component_name": "LLM",
                     "params": {
                       "llm_id": "deepseek-chat",
-                      "sys_prompt": "You are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\" Answers need to consider chat history.\n      Here is the knowledge base:\n      {retrieval:0@formalized_content}\n      The above is the knowledge base.",
+                      "sys_prompt": "You are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\" Answers need to consider chat history.\n      Here is the knowledge base:\n      {retrieval:0@formalized_content}\n      Above is the knowledge base.",
                       "temperature": 0.2
                     }
                 },
@@ -51,7 +51,7 @@
   },
   "history": [],
   "path": [],
-  "retrival": {"chunks": [], "doc_aggs": []},
+  "retrieval": {"chunks": [], "doc_aggs": []},
   "globals": {
     "sys.query": "",
     "sys.user_id": "",

agent/test/dsl_examples/retrieval_categorize_and_generate.json

@@ -65,7 +65,7 @@
                     "component_name": "Agent",
                     "params": {
                       "llm_id": "deepseek-chat",
-                      "sys_prompt": "You are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\" Answers need to consider chat history.\n      Here is the knowledge base:\n      {retrieval:0@formalized_content}\n      The above is the knowledge base.",
+                      "sys_prompt": "You are an intelligent assistant. Please summarize the content of the dataset to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\" Answers need to consider chat history.\n      Here is the knowledge base:\n      {retrieval:0@formalized_content}\n      The above is the knowledge base.",
                       "temperature": 0.2
                     }
                 },
@@ -85,7 +85,7 @@
   },
   "history": [],
   "path": [],
-  "retrival": {"chunks": [], "doc_aggs": []},
+  "retrieval": {"chunks": [], "doc_aggs": []},
   "globals": {
     "sys.query": "",
     "sys.user_id": "",

agent/test/dsl_examples/tavily_and_generate.json

@@ -25,7 +25,7 @@
                     "component_name": "LLM",
                     "params": {
                       "llm_id": "deepseek-chat",
-                      "sys_prompt": "You are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\" Answers need to consider chat history.\n      Here is the knowledge base:\n      {tavily:0@formalized_content}\n      The above is the knowledge base.",
+                      "sys_prompt": "You are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\" Answers need to consider chat history.\n      Here is the knowledge base:\n      {tavily:0@formalized_content}\n      Above is the knowledge base.",
                       "temperature": 0.2
                     }
                 },
@@ -45,7 +45,7 @@
   },
   "history": [],
   "path": [],
-  "retrival": {"chunks": [], "doc_aggs": []},
+  "retrieval": {"chunks": [], "doc_aggs": []},
   "globals": {
     "sys.query": "",
     "sys.user_id": "",

agent/tools/retrieval.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import asyncio
 from functools import partial
 import json
 import os
@@ -21,13 +22,15 @@ from abc import ABC
 from agent.tools.base import ToolParamBase, ToolBase, ToolMeta
 from common.constants import LLMType
 from api.db.services.document_service import DocumentService
-from api.db.services.dialog_service import meta_filter
+from common.metadata_utils import apply_meta_data_filter
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
+from api.db.services.memory_service import MemoryService
+from api.db.joint_services import memory_message_service
 from common import settings
 from common.connection_utils import timeout
 from rag.app.tag import label_question
-from rag.prompts.generator import cross_languages, kb_prompt, gen_meta_filter
+from rag.prompts.generator import cross_languages, kb_prompt, memory_prompt
 
 
 class RetrievalParam(ToolParamBase):
@@ -56,6 +59,7 @@ class RetrievalParam(ToolParamBase):
         self.top_n = 8
         self.top_k = 1024
         self.kb_ids = []
+        self.memory_ids = []
         self.kb_vars = []
         self.rerank_id = ""
         self.empty_response = ""
@@ -80,15 +84,7 @@ class RetrievalParam(ToolParamBase):
 class Retrieval(ToolBase, ABC):
     component_name = "Retrieval"
 
-    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
-    def _invoke(self, **kwargs):
-        if self.check_if_canceled("Retrieval processing"):
-            return
-
-        if not kwargs.get("query"):
-            self.set_output("formalized_content", self._param.empty_response)
-            return
-
+    async def _retrieve_kb(self, query_text: str):
         kb_ids: list[str] = []
         for id in self._param.kb_ids:
             if id.find("@") < 0:
@@ -123,54 +119,58 @@ class Retrieval(ToolBase, ABC):
         if self._param.rerank_id:
             rerank_mdl = LLMBundle(kbs[0].tenant_id, LLMType.RERANK, self._param.rerank_id)
 
-        vars = self.get_input_elements_from_text(kwargs["query"])
-        vars = {k:o["value"] for k,o in vars.items()}
-        query = self.string_format(kwargs["query"], vars)
+        vars = self.get_input_elements_from_text(query_text)
+        vars = {k: o["value"] for k, o in vars.items()}
+        query = self.string_format(query_text, vars)
 
-        doc_ids=[]
-        if self._param.meta_data_filter!={}:
+        doc_ids = []
+        if self._param.meta_data_filter != {}:
             metas = DocumentService.get_meta_by_kbs(kb_ids)
-            if self._param.meta_data_filter.get("method") == "auto":
+
+            def _resolve_manual_filter(flt: dict) -> dict:
+                pat = re.compile(self.variable_ref_patt)
+                s = flt.get("value", "")
+                out_parts = []
+                last = 0
+
+                for m in pat.finditer(s):
+                    out_parts.append(s[last:m.start()])
+                    key = m.group(1)
+                    v = self._canvas.get_variable_value(key)
+                    if v is None:
+                        rep = ""
+                    elif isinstance(v, partial):
+                        buf = []
+                        for chunk in v():
+                            buf.append(chunk)
+                        rep = "".join(buf)
+                    elif isinstance(v, str):
+                        rep = v
+                    else:
+                        rep = json.dumps(v, ensure_ascii=False)
+
+                    out_parts.append(rep)
+                    last = m.end()
+
+                out_parts.append(s[last:])
+                flt["value"] = "".join(out_parts)
+                return flt
+
+            chat_mdl = None
+            if self._param.meta_data_filter.get("method") in ["auto", "semi_auto"]:
                 chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT)
-                filters: dict = gen_meta_filter(chat_mdl, metas, query)
-                doc_ids.extend(meta_filter(metas, filters["conditions"], filters.get("logic", "and")))
-                if not doc_ids:
-                    doc_ids = None
-            elif self._param.meta_data_filter.get("method") == "manual":
-                filters = self._param.meta_data_filter["manual"]
-                for flt in filters:
-                    pat = re.compile(self.variable_ref_patt)
-                    s = flt["value"]
-                    out_parts = []
-                    last = 0
 
-                    for m in pat.finditer(s):
-                        out_parts.append(s[last:m.start()])
-                        key = m.group(1)
-                        v = self._canvas.get_variable_value(key)
-                        if v is None:
-                            rep = ""
-                        elif isinstance(v, partial):
-                            buf = []
-                            for chunk in v():
-                                buf.append(chunk)
-                            rep = "".join(buf)
-                        elif isinstance(v, str):
-                            rep = v
-                        else:
-                            rep = json.dumps(v, ensure_ascii=False)
-
-                        out_parts.append(rep)
-                        last = m.end()
-
-                    out_parts.append(s[last:])
-                    flt["value"] = "".join(out_parts)
-                doc_ids.extend(meta_filter(metas, filters, self._param.meta_data_filter.get("logic", "and")))
-                if filters and not doc_ids:
-                    doc_ids = ["-999"]
+            doc_ids = await apply_meta_data_filter(
+                self._param.meta_data_filter,
+                metas,
+                query,
+                chat_mdl,
+                doc_ids,
+                _resolve_manual_filter if self._param.meta_data_filter.get("method") == "manual" else None,
+            )
 
         if self._param.cross_languages:
-            query = cross_languages(kbs[0].tenant_id, None, query, self._param.cross_languages)
+            query = await cross_languages(kbs[0].tenant_id, None, query, self._param.cross_languages)
 
         if kbs:
             query = re.sub(r"^user[:：\s]*", "", query, flags=re.IGNORECASE)
@@ -193,18 +193,20 @@ class Retrieval(ToolBase, ABC):
 
             if self._param.toc_enhance:
                 chat_mdl = LLMBundle(self._canvas._tenant_id, LLMType.CHAT)
-                cks = settings.retriever.retrieval_by_toc(query, kbinfos["chunks"], [kb.tenant_id for kb in kbs], chat_mdl, self._param.top_n)
+                cks = settings.retriever.retrieval_by_toc(query, kbinfos["chunks"], [kb.tenant_id for kb in kbs],
+                                                          chat_mdl, self._param.top_n)
                 if self.check_if_canceled("Retrieval processing"):
                     return
                 if cks:
                     kbinfos["chunks"] = cks
-            kbinfos["chunks"] = settings.retriever.retrieval_by_children(kbinfos["chunks"], [kb.tenant_id for kb in kbs])
+            kbinfos["chunks"] = settings.retriever.retrieval_by_children(kbinfos["chunks"],
+                                                                         [kb.tenant_id for kb in kbs])
             if self._param.use_kg:
                 ck = settings.kg_retriever.retrieval(query,
-                                                       [kb.tenant_id for kb in kbs],
-                                                       kb_ids,
-                                                       embd_mdl,
-                                                       LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT))
+                                                     [kb.tenant_id for kb in kbs],
+                                                     kb_ids,
+                                                     embd_mdl,
+                                                     LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT))
                 if self.check_if_canceled("Retrieval processing"):
                     return
                 if ck["content_with_weight"]:
@@ -213,7 +215,8 @@ class Retrieval(ToolBase, ABC):
             kbinfos = {"chunks": [], "doc_aggs": []}
 
         if self._param.use_kg and kbs:
-            ck = settings.kg_retriever.retrieval(query, [kb.tenant_id for kb in kbs], filtered_kb_ids, embd_mdl, LLMBundle(kbs[0].tenant_id, LLMType.CHAT))
+            ck = settings.kg_retriever.retrieval(query, [kb.tenant_id for kb in kbs], filtered_kb_ids, embd_mdl,
+                                                 LLMBundle(kbs[0].tenant_id, LLMType.CHAT))
             if self.check_if_canceled("Retrieval processing"):
                 return
             if ck["content_with_weight"]:
@@ -243,6 +246,58 @@ class Retrieval(ToolBase, ABC):
 
         return form_cnt
 
+    async def _retrieve_memory(self, query_text: str):
+        memory_ids: list[str] = [memory_id for memory_id in self._param.memory_ids]
+        memory_list = MemoryService.get_by_ids(memory_ids)
+        if not memory_list:
+            raise Exception("No memory is selected.")
+
+        embd_names = list({memory.embd_id for memory in memory_list})
+        assert len(embd_names) == 1, "Memory use different embedding models."
+
+        vars = self.get_input_elements_from_text(query_text)
+        vars = {k: o["value"] for k, o in vars.items()}
+        query = self.string_format(query_text, vars)
+        # query message
+        message_list = memory_message_service.query_message({"memory_id": memory_ids}, {
+            "query": query,
+            "similarity_threshold": self._param.similarity_threshold,
+            "keywords_similarity_weight": self._param.keywords_similarity_weight,
+            "top_n": self._param.top_n
+        })
+        if not message_list:
+            self.set_output("formalized_content", self._param.empty_response)
+            return ""
+        formated_content = "\n".join(memory_prompt(message_list, 200000))
+        # set formalized_content output
+        self.set_output("formalized_content", formated_content)
+
+        return formated_content
+
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
+    async def _invoke_async(self, **kwargs):
+        if self.check_if_canceled("Retrieval processing"):
+            return
+        if not kwargs.get("query"):
+            self.set_output("formalized_content", self._param.empty_response)
+            return
+
+        if hasattr(self._param, "retrieval_from") and self._param.retrieval_from == "dataset":
+            return await self._retrieve_kb(kwargs["query"])
+        elif hasattr(self._param, "retrieval_from") and self._param.retrieval_from == "memory":
+            return await self._retrieve_memory(kwargs["query"])
+        elif self._param.kb_ids:
+            return await self._retrieve_kb(kwargs["query"])
+        elif hasattr(self._param, "memory_ids") and self._param.memory_ids:
+            return await self._retrieve_memory(kwargs["query"])
+        else:
+            self.set_output("formalized_content", self._param.empty_response)
+            return
+
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
+    def _invoke(self, **kwargs):
+        return asyncio.run(self._invoke_async(**kwargs))
+
     def thoughts(self) -> str:
         return """
 Keywords: {}

agent/tools/yahoofinance.py

@@ -75,7 +75,7 @@ class YahooFinance(ToolBase, ABC):
     @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 60)))
     def _invoke(self, **kwargs):
         if self.check_if_canceled("YahooFinance processing"):
-            return
+            return None
 
         if not kwargs.get("stock_code"):
             self.set_output("report", "")
@@ -84,33 +84,33 @@ class YahooFinance(ToolBase, ABC):
         last_e = ""
         for _ in range(self._param.max_retries+1):
             if self.check_if_canceled("YahooFinance processing"):
-                return
+                return None
 
-            yohoo_res = []
+            yahoo_res = []
             try:
                 msft = yf.Ticker(kwargs["stock_code"])
                 if self.check_if_canceled("YahooFinance processing"):
-                    return
+                    return None
 
                 if self._param.info:
-                    yohoo_res.append("# Information:\n" + pd.Series(msft.info).to_markdown() + "\n")
+                    yahoo_res.append("# Information:\n" + pd.Series(msft.info).to_markdown() + "\n")
                 if self._param.history:
-                    yohoo_res.append("# History:\n" + msft.history().to_markdown() + "\n")
+                    yahoo_res.append("# History:\n" + msft.history().to_markdown() + "\n")
                 if self._param.financials:
-                    yohoo_res.append("# Calendar:\n" + pd.DataFrame(msft.calendar).to_markdown() + "\n")
+                    yahoo_res.append("# Calendar:\n" + pd.DataFrame(msft.calendar).to_markdown() + "\n")
                 if self._param.balance_sheet:
-                    yohoo_res.append("# Balance sheet:\n" + msft.balance_sheet.to_markdown() + "\n")
-                    yohoo_res.append("# Quarterly balance sheet:\n" + msft.quarterly_balance_sheet.to_markdown() + "\n")
+                    yahoo_res.append("# Balance sheet:\n" + msft.balance_sheet.to_markdown() + "\n")
+                    yahoo_res.append("# Quarterly balance sheet:\n" + msft.quarterly_balance_sheet.to_markdown() + "\n")
                 if self._param.cash_flow_statement:
-                    yohoo_res.append("# Cash flow statement:\n" + msft.cashflow.to_markdown() + "\n")
-                    yohoo_res.append("# Quarterly cash flow statement:\n" + msft.quarterly_cashflow.to_markdown() + "\n")
+                    yahoo_res.append("# Cash flow statement:\n" + msft.cashflow.to_markdown() + "\n")
+                    yahoo_res.append("# Quarterly cash flow statement:\n" + msft.quarterly_cashflow.to_markdown() + "\n")
                 if self._param.news:
-                    yohoo_res.append("# News:\n" + pd.DataFrame(msft.news).to_markdown() + "\n")
-                self.set_output("report", "\n\n".join(yohoo_res))
+                    yahoo_res.append("# News:\n" + pd.DataFrame(msft.news).to_markdown() + "\n")
+                self.set_output("report", "\n\n".join(yahoo_res))
                 return self.output("report")
             except Exception as e:
                 if self.check_if_canceled("YahooFinance processing"):
-                    return
+                    return None
 
                 last_e = e
                 logging.exception(f"YahooFinance error: {e}")

agentic_reasoning/deep_research.py

@@ -51,7 +51,7 @@ class DeepResearcher:
         """Remove Result Tags"""
         return DeepResearcher._remove_tags(text, BEGIN_SEARCH_RESULT, END_SEARCH_RESULT)
 
-    def _generate_reasoning(self, msg_history):
+    async def _generate_reasoning(self, msg_history):
         """Generate reasoning steps"""
         query_think = ""
         if msg_history[-1]["role"] != "user":
@@ -59,13 +59,14 @@ class DeepResearcher:
         else:
             msg_history[-1]["content"] += "\n\nContinues reasoning with the new information.\n"
             
-        for ans in self.chat_mdl.chat_streamly(REASON_PROMPT, msg_history, {"temperature": 0.7}):
+        async for ans in self.chat_mdl.async_chat_streamly(REASON_PROMPT, msg_history, {"temperature": 0.7}):
             ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
             if not ans:
                 continue
             query_think = ans
             yield query_think
-        return query_think
+            query_think = ""
+        yield query_think
 
     def _extract_search_queries(self, query_think, question, step_index):
         """Extract search queries from thinking"""
@@ -143,10 +144,10 @@ class DeepResearcher:
                 if d["doc_id"] not in dids:
                     chunk_info["doc_aggs"].append(d)
 
-    def _extract_relevant_info(self, truncated_prev_reasoning, search_query, kbinfos):
+    async def _extract_relevant_info(self, truncated_prev_reasoning, search_query, kbinfos):
         """Extract and summarize relevant information"""
         summary_think = ""
-        for ans in self.chat_mdl.chat_streamly(
+        async for ans in self.chat_mdl.async_chat_streamly(
                 RELEVANT_EXTRACTION_PROMPT.format(
                     prev_reasoning=truncated_prev_reasoning,
                     search_query=search_query,
@@ -160,10 +161,11 @@ class DeepResearcher:
                 continue
             summary_think = ans
             yield summary_think
+            summary_think = ""
         
-        return summary_think
+        yield summary_think
 
-    def thinking(self, chunk_info: dict, question: str):
+    async def thinking(self, chunk_info: dict, question: str):
         executed_search_queries = []
         msg_history = [{"role": "user", "content": f'Question:\"{question}\"\n'}]
         all_reasoning_steps = []
@@ -180,7 +182,7 @@ class DeepResearcher:
 
             # Step 1: Generate reasoning
             query_think = ""
-            for ans in self._generate_reasoning(msg_history):
+            async for ans in self._generate_reasoning(msg_history):
                 query_think = ans
                 yield {"answer": think + self._remove_query_tags(query_think) + "</think>", "reference": {}, "audio_binary": None}
 
@@ -223,7 +225,7 @@ class DeepResearcher:
                 # Step 6: Extract relevant information
                 think += "\n\n"
                 summary_think = ""
-                for ans in self._extract_relevant_info(truncated_prev_reasoning, search_query, kbinfos):
+                async for ans in self._extract_relevant_info(truncated_prev_reasoning, search_query, kbinfos):
                     summary_think = ans
                     yield {"answer": think + self._remove_result_tags(summary_think) + "</think>", "reference": {}, "audio_binary": None}
 

api/apps/__init__.py

@@ -28,7 +28,6 @@ from api.db.services import UserService
 from api.utils.json_encode import CustomJSONEncoder
 from api.utils import commands
 
-from flask_mail import Mail
 from quart_auth import Unauthorized
 from common import settings
 from api.utils.api_utils import server_error_response
@@ -39,10 +38,8 @@ settings.init_settings()
 
 __all__ = ["app"]
 
-
 app = Quart(__name__)
 app = cors(app, allow_origin="*")
-smtp_mail_server = Mail()
 
 # Add this at the beginning of your file to configure Swagger UI
 swagger_config = {
@@ -105,12 +102,13 @@ from werkzeug.local import LocalProxy
 T = TypeVar("T")
 P = ParamSpec("P")
 
+
 def _load_user():
     jwt = Serializer(secret_key=settings.SECRET_KEY)
     authorization = request.headers.get("Authorization")
     g.user = None
     if not authorization:
-        return
+        return None
 
     try:
         access_token = str(jwt.loads(authorization))
@@ -166,7 +164,7 @@ def login_required(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]
 
     @wraps(func)
     async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
-        if not current_user:# or not session.get("_user_id"):
+        if not current_user:  # or not session.get("_user_id"):
             raise Unauthorized()
         else:
             return await current_app.ensure_async(func)(*args, **kwargs)
@@ -180,7 +178,7 @@ def login_user(user, remember=False, duration=None, force=False, fresh=True):
     user's `is_active` property is ``False``, they will not be logged in
     unless `force` is ``True``.
 
-    This will return ``True`` if the log in attempt succeeds, and ``False`` if
+    This will return ``True`` if the login attempt succeeds, and ``False`` if
     it fails (i.e. because the user is inactive).
 
     :param user: The user object to log in.
@@ -230,6 +228,7 @@ def logout_user():
 
     return True
 
+
 def search_pages_path(page_path):
     app_path_list = [
         path for path in page_path.glob("*_app.py") if not path.name.startswith(".")
@@ -276,6 +275,16 @@ client_urls_prefix = [
 ]
 
 
+@app.errorhandler(404)
+async def not_found(error):
+    error_msg: str = f"The requested URL {request.path} was not found"
+    logging.error(error_msg)
+    return {
+        "error": "Not Found",
+        "message": error_msg,
+    }, 404
+
+
 @app.teardown_request
 def _db_close(exception):
     if exception:

api/apps/canvas_app.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 #
 import asyncio
+import inspect
 import json
 import logging
 from functools import partial
@@ -146,13 +147,13 @@ async def run():
     if cvs.canvas_category == CanvasCategory.DataFlow:
         task_id = get_uuid()
         Pipeline(cvs.dsl, tenant_id=current_user.id, doc_id=CANVAS_DEBUG_DOC_ID, task_id=task_id, flow_id=req["id"])
-        ok, error_message = await asyncio.to_thread(queue_dataflow, user_id, req["id"], task_id, files[0], 0)
+        ok, error_message = await asyncio.to_thread(queue_dataflow, user_id, req["id"], task_id, CANVAS_DEBUG_DOC_ID, files[0], 0)
         if not ok:
             return get_data_error_result(message=error_message)
         return get_json_result(data={"message_id": task_id})
 
     try:
-        canvas = Canvas(cvs.dsl, current_user.id)
+        canvas = Canvas(cvs.dsl, current_user.id, canvas_id=cvs.id)
     except Exception as e:
         return server_error_response(e)
 
@@ -191,7 +192,7 @@ async def rerun():
     if 0 < doc["progress"] < 1:
         return get_data_error_result(message=f"`{doc['name']}` is processing...")
 
-    if settings.docStoreConn.indexExist(search.index_name(current_user.id), doc["kb_id"]):
+    if settings.docStoreConn.index_exist(search.index_name(current_user.id), doc["kb_id"]):
         settings.docStoreConn.delete({"doc_id": doc["id"]}, search.index_name(current_user.id), doc["kb_id"])
     doc["progress_msg"] = ""
     doc["chunk_num"] = 0
@@ -231,7 +232,7 @@ async def reset():
         if not e:
             return get_data_error_result(message="canvas not found.")
 
-        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id)
+        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id, canvas_id=user_canvas.id)
         canvas.reset()
         req["dsl"] = json.loads(str(canvas))
         UserCanvasService.update_by_id(req["id"], {"dsl": req["dsl"]})
@@ -269,7 +270,7 @@ def input_form():
                 data=False, message='Only owner of canvas authorized for this operation.',
                 code=RetCode.OPERATING_ERROR)
 
-        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id)
+        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id, canvas_id=user_canvas.id)
         return get_json_result(data=canvas.get_component_input_form(cpn_id))
     except Exception as e:
         return server_error_response(e)
@@ -286,7 +287,7 @@ async def debug():
             code=RetCode.OPERATING_ERROR)
     try:
         e, user_canvas = UserCanvasService.get_by_id(req["id"])
-        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id)
+        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id, canvas_id=user_canvas.id)
         canvas.reset()
         canvas.message_id = get_uuid()
         component = canvas.get_component(req["component_id"])["obj"]
@@ -299,8 +300,13 @@ async def debug():
         for k in outputs.keys():
             if isinstance(outputs[k], partial):
                 txt = ""
-                for c in outputs[k]():
-                    txt += c
+                iter_obj = outputs[k]()
+                if inspect.isasyncgen(iter_obj):
+                    async for c in iter_obj:
+                        txt += c
+                else:
+                    for c in iter_obj:
+                        txt += c
                 outputs[k] = txt
         return get_json_result(data=outputs)
     except Exception as e:
@@ -342,7 +348,15 @@ async def test_db_connect():
                 f"UID={req['username']};"
                 f"PWD={req['password']};"
             )
-            logging.info(conn_str)
+            redacted_conn_str = (
+                f"DATABASE={req['database']};"
+                f"HOSTNAME={req['host']};"
+                f"PORT={req['port']};"
+                f"PROTOCOL=TCPIP;"
+                f"UID={req['username']};"
+                f"PWD=****;"
+            )
+            logging.info(redacted_conn_str)
             conn = ibm_db.connect(conn_str, "", "")
             stmt = ibm_db.exec_immediate(conn, "SELECT 1 FROM sysibm.sysdummy1")
             ibm_db.fetch_assoc(stmt)

api/apps/chunk_app.py

@@ -17,14 +17,14 @@ import asyncio
 import datetime
 import json
 import re
-
+import base64
 import xxhash
 from quart import request
 
-from api.db.services.dialog_service import meta_filter
 from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
+from common.metadata_utils import apply_meta_data_filter
 from api.db.services.search_service import SearchService
 from api.db.services.user_service import UserTenantService
 from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request, \
@@ -32,7 +32,7 @@ from api.utils.api_utils import get_data_error_result, get_json_result, server_e
 from rag.app.qa import beAdoc, rmPrefix
 from rag.app.tag import label_question
 from rag.nlp import rag_tokenizer, search
-from rag.prompts.generator import gen_meta_filter, cross_languages, keyword_extraction
+from rag.prompts.generator import cross_languages, keyword_extraction
 from common.string_utils import remove_redundant_spaces
 from common.constants import RetCode, LLMType, ParserType, PAGERANK_FLD
 from common import settings
@@ -76,6 +76,7 @@ async def list_chunk():
                 "image_id": sres.field[id].get("img_id", ""),
                 "available_int": int(sres.field[id].get("available_int", 1)),
                 "positions": sres.field[id].get("position_int", []),
+                "doc_type_kwd": sres.field[id].get("doc_type_kwd")
             }
             assert isinstance(d["positions"], list)
             assert len(d["positions"]) == 0 or (isinstance(d["positions"][0], list) and len(d["positions"][0]) == 5)
@@ -174,6 +175,13 @@ async def set():
             v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
             _d["q_%d_vec" % len(v)] = v.tolist()
             settings.docStoreConn.update({"id": req["chunk_id"]}, _d, search.index_name(tenant_id), doc.kb_id)
+
+            # update image
+            image_base64 = req.get("image_base64", None)
+            if image_base64:
+                bkt, name = req.get("img_id", "-").split("-")
+                image_binary = base64.b64decode(image_base64)
+                settings.STORAGE_IMPL.put(bkt, name, image_binary)
             return get_json_result(data=True)
 
         return await asyncio.to_thread(_set_sync)
@@ -313,24 +321,25 @@ async def retrieval_test():
     langs = req.get("cross_languages", [])
     user_id = current_user.id
 
-    def _retrieval_sync():
+    async def _retrieval():
         local_doc_ids = list(doc_ids) if doc_ids else []
         tenant_ids = []
 
+        meta_data_filter = {}
+        chat_mdl = None
         if req.get("search_id", ""):
             search_config = SearchService.get_detail(req.get("search_id", "")).get("search_config", {})
             meta_data_filter = search_config.get("meta_data_filter", {})
-            metas = DocumentService.get_meta_by_kbs(kb_ids)
-            if meta_data_filter.get("method") == "auto":
+            if meta_data_filter.get("method") in ["auto", "semi_auto"]:
                 chat_mdl = LLMBundle(user_id, LLMType.CHAT, llm_name=search_config.get("chat_id", ""))
-                filters: dict = gen_meta_filter(chat_mdl, metas, question)
-                local_doc_ids.extend(meta_filter(metas, filters["conditions"], filters.get("logic", "and")))
-                if not local_doc_ids:
-                    local_doc_ids = None
-            elif meta_data_filter.get("method") == "manual":
-                local_doc_ids.extend(meta_filter(metas, meta_data_filter["manual"], meta_data_filter.get("logic", "and")))
-                if meta_data_filter["manual"] and not local_doc_ids:
-                    local_doc_ids = ["-999"]
+        else:
+            meta_data_filter = req.get("meta_data_filter") or {}
+            if meta_data_filter.get("method") in ["auto", "semi_auto"]:
+                chat_mdl = LLMBundle(user_id, LLMType.CHAT)
+
+        if meta_data_filter:
+            metas = DocumentService.get_meta_by_kbs(kb_ids)
+            local_doc_ids = await apply_meta_data_filter(meta_data_filter, metas, question, chat_mdl, local_doc_ids)
 
         tenants = UserTenantService.query(user_id=user_id)
         for kb_id in kb_ids:
@@ -341,7 +350,7 @@ async def retrieval_test():
                     break
             else:
                 return get_json_result(
-                    data=False, message='Only owner of knowledgebase authorized for this operation.',
+                    data=False, message='Only owner of dataset authorized for this operation.',
                     code=RetCode.OPERATING_ERROR)
 
         e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
@@ -350,7 +359,7 @@ async def retrieval_test():
 
         _question = question
         if langs:
-            _question = cross_languages(kb.tenant_id, None, _question, langs)
+            _question = await cross_languages(kb.tenant_id, None, _question, langs)
 
         embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
 
@@ -360,7 +369,7 @@ async def retrieval_test():
 
         if req.get("keyword", False):
             chat_mdl = LLMBundle(kb.tenant_id, LLMType.CHAT)
-            _question += keyword_extraction(chat_mdl, _question)
+            _question += await keyword_extraction(chat_mdl, _question)
 
         labels = label_question(_question, [kb])
         ranks = settings.retriever.retrieval(_question, embd_mdl, tenant_ids, kb_ids, page, size,
@@ -379,6 +388,7 @@ async def retrieval_test():
                                                    LLMBundle(kb.tenant_id, LLMType.CHAT))
             if ck["content_with_weight"]:
                 ranks["chunks"].insert(0, ck)
+        ranks["chunks"] = settings.retriever.retrieval_by_children(ranks["chunks"], tenant_ids)
 
         for c in ranks["chunks"]:
             c.pop("vector", None)
@@ -387,7 +397,7 @@ async def retrieval_test():
         return get_json_result(data=ranks)
 
     try:
-        return await asyncio.to_thread(_retrieval_sync)
+        return await _retrieval()
     except Exception as e:
         if str(e).find("not_found") > 0:
             return get_json_result(data=False, message='No chunk found! Check the chunk status please!',

api/apps/connector_app.py

@@ -28,11 +28,12 @@ from api.db import InputType
 from api.db.services.connector_service import ConnectorService, SyncLogsService
 from api.utils.api_utils import get_data_error_result, get_json_result, get_request_json, validate_request
 from common.constants import RetCode, TaskStatus
-from common.data_source.config import GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI, GMAIL_WEB_OAUTH_REDIRECT_URI, DocumentSource
-from common.data_source.google_util.constant import GOOGLE_WEB_OAUTH_POPUP_TEMPLATE, GOOGLE_SCOPES
+from common.data_source.config import GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI, GMAIL_WEB_OAUTH_REDIRECT_URI, BOX_WEB_OAUTH_REDIRECT_URI, DocumentSource
+from common.data_source.google_util.constant import WEB_OAUTH_POPUP_TEMPLATE, GOOGLE_SCOPES
 from common.misc_utils import get_uuid
 from rag.utils.redis_conn import REDIS_CONN
 from api.apps import login_required, current_user
+from box_sdk_gen import BoxOAuth, OAuthConfig, GetAuthorizeUrlOptions
 
 
 @manager.route("/set", methods=["POST"])  # noqa: F821
@@ -117,8 +118,6 @@ def rm_connector(connector_id):
     return get_json_result(data=True)
 
 
-GOOGLE_WEB_FLOW_STATE_PREFIX = "google_drive_web_flow_state"
-GOOGLE_WEB_FLOW_RESULT_PREFIX = "google_drive_web_flow_result"
 WEB_FLOW_TTL_SECS = 15 * 60
 
 
@@ -129,10 +128,7 @@ def _web_state_cache_key(flow_id: str, source_type: str | None = None) -> str:
     When source_type == "gmail", a different prefix is used so that
     Drive/Gmail flows don't clash in Redis.
     """
-    if source_type == "gmail":
-        prefix = "gmail_web_flow_state"
-    else:
-        prefix = GOOGLE_WEB_FLOW_STATE_PREFIX
+    prefix = f"{source_type}_web_flow_state"
     return f"{prefix}:{flow_id}"
 
 
@@ -141,10 +137,7 @@ def _web_result_cache_key(flow_id: str, source_type: str | None = None) -> str:
 
     Mirrors _web_state_cache_key logic for result storage.
     """
-    if source_type == "gmail":
-        prefix = "gmail_web_flow_result"
-    else:
-        prefix = GOOGLE_WEB_FLOW_RESULT_PREFIX
+    prefix = f"{source_type}_web_flow_result"
     return f"{prefix}:{flow_id}"
 
 
@@ -180,7 +173,7 @@ async def _render_web_oauth_popup(flow_id: str, success: bool, message: str, sou
         }
     )
     # TODO(google-oauth): title/heading/message may need to reflect drive/gmail based on cached type
-    html = GOOGLE_WEB_OAUTH_POPUP_TEMPLATE.format(
+    html = WEB_OAUTH_POPUP_TEMPLATE.format(
         title=f"Google {source.capitalize()} Authorization",
         heading="Authorization complete" if success else "Authorization failed",
         message=escaped_message,
@@ -204,8 +197,8 @@ async def start_google_web_oauth():
         redirect_uri = GMAIL_WEB_OAUTH_REDIRECT_URI
         scopes = GOOGLE_SCOPES[DocumentSource.GMAIL]
     else:
-        redirect_uri = GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI if source == "google-drive" else GMAIL_WEB_OAUTH_REDIRECT_URI
-        scopes = GOOGLE_SCOPES[DocumentSource.GOOGLE_DRIVE if source == "google-drive" else DocumentSource.GMAIL]
+        redirect_uri = GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI
+        scopes = GOOGLE_SCOPES[DocumentSource.GOOGLE_DRIVE]
 
     if not redirect_uri:
         return get_json_result(
@@ -271,8 +264,6 @@ async def google_gmail_web_oauth_callback():
     state_id = request.args.get("state")
     error = request.args.get("error")
     source = "gmail"
-    if source != 'gmail':
-        return await _render_web_oauth_popup("", False, "Invalid Google OAuth type.", source)
 
     error_description = request.args.get("error_description") or error
 
@@ -313,9 +304,6 @@ async def google_gmail_web_oauth_callback():
         "credentials": creds_json,
     }
     REDIS_CONN.set_obj(_web_result_cache_key(state_id, source), result_payload, WEB_FLOW_TTL_SECS)
-
-    print("\n\n", _web_result_cache_key(state_id, source), "\n\n")
-
     REDIS_CONN.delete(_web_state_cache_key(state_id, source))
 
     return await _render_web_oauth_popup(state_id, True, "Authorization completed successfully.", source)
@@ -326,8 +314,6 @@ async def google_drive_web_oauth_callback():
     state_id = request.args.get("state")
     error = request.args.get("error")
     source = "google-drive"
-    if source not in ("google-drive", "gmail"):
-        return await _render_web_oauth_popup("", False, "Invalid Google OAuth type.", source)
 
     error_description = request.args.get("error_description") or error
 
@@ -391,3 +377,107 @@ async def poll_google_web_result():
 
     REDIS_CONN.delete(_web_result_cache_key(flow_id, source))
     return get_json_result(data={"credentials": result.get("credentials")})
+
+@manager.route("/box/oauth/web/start", methods=["POST"])  # noqa: F821
+@login_required
+async def start_box_web_oauth():
+    req = await get_request_json()
+
+    client_id = req.get("client_id")
+    client_secret = req.get("client_secret")    
+    redirect_uri = req.get("redirect_uri", BOX_WEB_OAUTH_REDIRECT_URI)
+
+    if not client_id or not client_secret:
+        return get_json_result(code=RetCode.ARGUMENT_ERROR, message="Box client_id and client_secret are required.")
+
+    flow_id = str(uuid.uuid4())
+
+    box_auth = BoxOAuth(
+        OAuthConfig(
+            client_id=client_id,
+            client_secret=client_secret,
+        )
+    )
+
+    auth_url = box_auth.get_authorize_url(
+        options=GetAuthorizeUrlOptions(
+            redirect_uri=redirect_uri,
+            state=flow_id,
+        )
+    )
+
+    cache_payload = {
+        "user_id": current_user.id,
+        "auth_url": auth_url,
+        "client_id": client_id,
+        "client_secret": client_secret,
+        "created_at": int(time.time()),
+    }
+    REDIS_CONN.set_obj(_web_state_cache_key(flow_id, "box"), cache_payload, WEB_FLOW_TTL_SECS)
+    return get_json_result(
+        data = {
+            "flow_id": flow_id,
+            "authorization_url": auth_url,
+            "expires_in": WEB_FLOW_TTL_SECS,}
+    )
+
+@manager.route("/box/oauth/web/callback", methods=["GET"])  # noqa: F821
+async def box_web_oauth_callback():
+    flow_id = request.args.get("state")
+    if not flow_id:
+        return await _render_web_oauth_popup("", False, "Missing OAuth parameters.", "box")
+    
+    code = request.args.get("code")
+    if not code:
+        return await _render_web_oauth_popup(flow_id, False, "Missing authorization code from Box.", "box")
+
+    cache_payload = json.loads(REDIS_CONN.get(_web_state_cache_key(flow_id, "box")))
+    if not cache_payload:
+        return get_json_result(code=RetCode.ARGUMENT_ERROR, message="Box OAuth session expired or invalid.")
+
+    error = request.args.get("error")
+    error_description = request.args.get("error_description") or error
+    if error:
+        REDIS_CONN.delete(_web_state_cache_key(flow_id, "box"))
+        return await _render_web_oauth_popup(flow_id, False, error_description or "Authorization failed.", "box")
+    
+    auth = BoxOAuth(
+        OAuthConfig(
+            client_id=cache_payload.get("client_id"),
+            client_secret=cache_payload.get("client_secret"),
+        )
+    )
+
+    auth.get_tokens_authorization_code_grant(code)
+    token = auth.retrieve_token()
+    result_payload = {
+        "user_id": cache_payload.get("user_id"),
+        "client_id": cache_payload.get("client_id"),
+        "client_secret": cache_payload.get("client_secret"),
+        "access_token": token.access_token,
+        "refresh_token": token.refresh_token,
+    }
+
+    REDIS_CONN.set_obj(_web_result_cache_key(flow_id, "box"), result_payload, WEB_FLOW_TTL_SECS)
+    REDIS_CONN.delete(_web_state_cache_key(flow_id, "box"))
+
+    return await _render_web_oauth_popup(flow_id, True, "Authorization completed successfully.", "box")
+
+@manager.route("/box/oauth/web/result", methods=["POST"])  # noqa: F821
+@login_required
+@validate_request("flow_id")
+async def poll_box_web_result():
+    req = await get_request_json()
+    flow_id = req.get("flow_id")
+
+    cache_blob = REDIS_CONN.get(_web_result_cache_key(flow_id, "box"))
+    if not cache_blob:
+        return get_json_result(code=RetCode.RUNNING, message="Authorization is still pending.")
+
+    cache_raw = json.loads(cache_blob)
+    if cache_raw.get("user_id") != current_user.id:
+        return get_json_result(code=RetCode.PERMISSION_ERROR, message="You are not allowed to access this authorization result.")
+    
+    REDIS_CONN.delete(_web_result_cache_key(flow_id, "box"))
+
+    return get_json_result(data={"credentials": cache_raw})

api/apps/conversation_app.py

@@ -23,7 +23,7 @@ from quart import Response, request
 from api.apps import current_user, login_required
 from api.db.db_models import APIToken
 from api.db.services.conversation_service import ConversationService, structure_answer
-from api.db.services.dialog_service import DialogService, ask, chat, gen_mindmap
+from api.db.services.dialog_service import DialogService, async_ask, async_chat, gen_mindmap
 from api.db.services.llm_service import LLMBundle
 from api.db.services.search_service import SearchService
 from api.db.services.tenant_llm_service import TenantLLMService
@@ -218,10 +218,10 @@ async def completion():
             dia.llm_setting = chat_model_config
 
         is_embedded = bool(chat_model_id)
-        def stream():
+        async def stream():
             nonlocal dia, msg, req, conv
             try:
-                for ans in chat(dia, msg, True, **req):
+                async for ans in async_chat(dia, msg, True, **req):
                     ans = structure_answer(conv, ans, message_id, conv.id)
                     yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
                 if not is_embedded:
@@ -241,7 +241,7 @@ async def completion():
 
         else:
             answer = None
-            for ans in chat(dia, msg, **req):
+            async for ans in async_chat(dia, msg, **req):
                 answer = structure_answer(conv, ans, message_id, conv.id)
                 if not is_embedded:
                     ConversationService.update_by_id(conv.id, conv.to_dict())
@@ -406,10 +406,10 @@ async def ask_about():
     if search_app:
         search_config = search_app.get("search_config", {})
 
-    def stream():
+    async def stream():
         nonlocal req, uid
         try:
-            for ans in ask(req["question"], req["kb_ids"], uid, search_config=search_config):
+            async for ans in async_ask(req["question"], req["kb_ids"], uid, search_config=search_config):
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
         except Exception as e:
             yield "data:" + json.dumps({"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}}, ensure_ascii=False) + "\n\n"
@@ -435,7 +435,7 @@ async def mindmap():
     kb_ids.extend(req["kb_ids"])
     kb_ids = list(set(kb_ids))
 
-    mind_map = gen_mindmap(req["question"], kb_ids, search_app.get("tenant_id", current_user.id), search_config)
+    mind_map = await gen_mindmap(req["question"], kb_ids, search_app.get("tenant_id", current_user.id), search_config)
     if "error" in mind_map:
         return server_error_response(Exception(mind_map["error"]))
     return get_json_result(data=mind_map)

api/apps/dialog_app.py

@@ -65,7 +65,7 @@ async def set_dialog():
 
     if not is_create:
         if not req.get("kb_ids", []) and not prompt_config.get("tavily_api_key") and "{knowledge}" in prompt_config['system']:
-            return get_data_error_result(message="Please remove `{knowledge}` in system prompt since no knowledge base / Tavily used here.")
+            return get_data_error_result(message="Please remove `{knowledge}` in system prompt since no dataset / Tavily used here.")
 
         for p in prompt_config["parameters"]:
             if p["optional"]:

api/apps/document_app.py

@@ -27,6 +27,7 @@ from api.db import VALID_FILE_TYPES, FileType
 from api.db.db_models import Task
 from api.db.services import duplicate_name
 from api.db.services.document_service import DocumentService, doc_upload_and_parse
+from common.metadata_utils import meta_filter, convert_conditions
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
@@ -69,7 +70,7 @@ async def upload():
 
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
-        raise LookupError("Can't find this knowledgebase!")
+        raise LookupError("Can't find this dataset!")
     if not check_kb_team_permission(kb, current_user.id):
         return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
@@ -98,7 +99,7 @@ async def web_crawl():
         return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
-        raise LookupError("Can't find this knowledgebase!")
+        raise LookupError("Can't find this dataset!")
     if check_kb_team_permission(kb, current_user.id):
         return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
 
@@ -168,10 +169,10 @@ async def create():
     try:
         e, kb = KnowledgebaseService.get_by_id(kb_id)
         if not e:
-            return get_data_error_result(message="Can't find this knowledgebase!")
+            return get_data_error_result(message="Can't find this dataset!")
 
         if DocumentService.query(name=req["name"], kb_id=kb_id):
-            return get_data_error_result(message="Duplicated document name in the same knowledgebase.")
+            return get_data_error_result(message="Duplicated document name in the same dataset.")
 
         kb_root_folder = FileService.get_kb_folder(kb.tenant_id)
         if not kb_root_folder:
@@ -218,7 +219,7 @@ async def list_docs():
         if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
             break
     else:
-        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
+        return get_json_result(data=False, message="Only owner of dataset authorized for this operation.", code=RetCode.OPERATING_ERROR)
     keywords = request.args.get("keywords", "")
 
     page_number = int(request.args.get("page", 0))
@@ -233,6 +234,10 @@ async def list_docs():
 
     req = await get_request_json()
 
+    return_empty_metadata = req.get("return_empty_metadata", False)
+    if isinstance(return_empty_metadata, str):
+        return_empty_metadata = return_empty_metadata.lower() == "true"
+
     run_status = req.get("run_status", [])
     if run_status:
         invalid_status = {s for s in run_status if s not in VALID_TASK_STATUS}
@@ -246,9 +251,74 @@ async def list_docs():
             return get_data_error_result(message=f"Invalid filter conditions: {', '.join(invalid_types)} type{'s' if len(invalid_types) > 1 else ''}")
 
     suffix = req.get("suffix", [])
+    metadata_condition = req.get("metadata_condition", {}) or {}
+    metadata = req.get("metadata", {}) or {}
+    if isinstance(metadata, dict) and metadata.get("empty_metadata"):
+        return_empty_metadata = True
+        metadata = {k: v for k, v in metadata.items() if k != "empty_metadata"}
+    if return_empty_metadata:
+        metadata_condition = {}
+        metadata = {}
+    else:
+        if metadata_condition and not isinstance(metadata_condition, dict):
+            return get_data_error_result(message="metadata_condition must be an object.")
+        if metadata and not isinstance(metadata, dict):
+            return get_data_error_result(message="metadata must be an object.")
+
+    doc_ids_filter = None
+    metas = None
+    if metadata_condition or metadata:
+        metas = DocumentService.get_flatted_meta_by_kbs([kb_id])
+
+    if metadata_condition:
+        doc_ids_filter = set(meta_filter(metas, convert_conditions(metadata_condition), metadata_condition.get("logic", "and")))
+        if metadata_condition.get("conditions") and not doc_ids_filter:
+            return get_json_result(data={"total": 0, "docs": []})
+
+    if metadata:
+        metadata_doc_ids = None
+        for key, values in metadata.items():
+            if not values:
+                continue
+            if not isinstance(values, list):
+                values = [values]
+            values = [str(v) for v in values if v is not None and str(v).strip()]
+            if not values:
+                continue
+            key_doc_ids = set()
+            for value in values:
+                key_doc_ids.update(metas.get(key, {}).get(value, []))
+            if metadata_doc_ids is None:
+                metadata_doc_ids = key_doc_ids
+            else:
+                metadata_doc_ids &= key_doc_ids
+            if not metadata_doc_ids:
+                return get_json_result(data={"total": 0, "docs": []})
+        if metadata_doc_ids is not None:
+            if doc_ids_filter is None:
+                doc_ids_filter = metadata_doc_ids
+            else:
+                doc_ids_filter &= metadata_doc_ids
+            if not doc_ids_filter:
+                return get_json_result(data={"total": 0, "docs": []})
+
+    if doc_ids_filter is not None:
+        doc_ids_filter = list(doc_ids_filter)
 
     try:
-        docs, tol = DocumentService.get_by_kb_id(kb_id, page_number, items_per_page, orderby, desc, keywords, run_status, types, suffix)
+        docs, tol = DocumentService.get_by_kb_id(
+            kb_id,
+            page_number,
+            items_per_page,
+            orderby,
+            desc,
+            keywords,
+            run_status,
+            types,
+            suffix,
+            doc_ids_filter,
+            return_empty_metadata=return_empty_metadata,
+        )
 
         if create_time_from or create_time_to:
             filtered_docs = []
@@ -282,7 +352,7 @@ async def get_filter():
         if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
             break
     else:
-        return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.", code=RetCode.OPERATING_ERROR)
+        return get_json_result(data=False, message="Only owner of dataset authorized for this operation.", code=RetCode.OPERATING_ERROR)
 
     keywords = req.get("keywords", "")
 
@@ -319,6 +389,107 @@ async def doc_infos():
     return get_json_result(data=list(docs.dicts()))
 
 
+@manager.route("/metadata/summary", methods=["POST"])  # noqa: F821
+@login_required
+async def metadata_summary():
+    req = await get_request_json()
+    kb_id = req.get("kb_id")
+    if not kb_id:
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
+
+    tenants = UserTenantService.query(user_id=current_user.id)
+    for tenant in tenants:
+        if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
+            break
+    else:
+        return get_json_result(data=False, message="Only owner of dataset authorized for this operation.", code=RetCode.OPERATING_ERROR)
+
+    try:
+        summary = DocumentService.get_metadata_summary(kb_id)
+        return get_json_result(data={"summary": summary})
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/metadata/update", methods=["POST"])  # noqa: F821
+@login_required
+async def metadata_update():
+    req = await get_request_json()
+    kb_id = req.get("kb_id")
+    if not kb_id:
+        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
+
+    tenants = UserTenantService.query(user_id=current_user.id)
+    for tenant in tenants:
+        if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
+            break
+    else:
+        return get_json_result(data=False, message="Only owner of dataset authorized for this operation.", code=RetCode.OPERATING_ERROR)
+
+    selector = req.get("selector", {}) or {}
+    updates = req.get("updates", []) or []
+    deletes = req.get("deletes", []) or []
+
+    if not isinstance(selector, dict):
+        return get_json_result(data=False, message="selector must be an object.", code=RetCode.ARGUMENT_ERROR)
+    if not isinstance(updates, list) or not isinstance(deletes, list):
+        return get_json_result(data=False, message="updates and deletes must be lists.", code=RetCode.ARGUMENT_ERROR)
+
+    metadata_condition = selector.get("metadata_condition", {}) or {}
+    if metadata_condition and not isinstance(metadata_condition, dict):
+        return get_json_result(data=False, message="metadata_condition must be an object.", code=RetCode.ARGUMENT_ERROR)
+
+    document_ids = selector.get("document_ids", []) or []
+    if document_ids and not isinstance(document_ids, list):
+        return get_json_result(data=False, message="document_ids must be a list.", code=RetCode.ARGUMENT_ERROR)
+
+    for upd in updates:
+        if not isinstance(upd, dict) or not upd.get("key") or "value" not in upd:
+            return get_json_result(data=False, message="Each update requires key and value.", code=RetCode.ARGUMENT_ERROR)
+    for d in deletes:
+        if not isinstance(d, dict) or not d.get("key"):
+            return get_json_result(data=False, message="Each delete requires key.", code=RetCode.ARGUMENT_ERROR)
+
+    kb_doc_ids = KnowledgebaseService.list_documents_by_ids([kb_id])
+    target_doc_ids = set(kb_doc_ids)
+    if document_ids:
+        invalid_ids = set(document_ids) - set(kb_doc_ids)
+        if invalid_ids:
+            return get_json_result(data=False, message=f"These documents do not belong to dataset {kb_id}: {', '.join(invalid_ids)}", code=RetCode.ARGUMENT_ERROR)
+        target_doc_ids = set(document_ids)
+
+    if metadata_condition:
+        metas = DocumentService.get_flatted_meta_by_kbs([kb_id])
+        filtered_ids = set(meta_filter(metas, convert_conditions(metadata_condition), metadata_condition.get("logic", "and")))
+        target_doc_ids = target_doc_ids & filtered_ids
+        if metadata_condition.get("conditions") and not target_doc_ids:
+            return get_json_result(data={"updated": 0, "matched_docs": 0})
+
+    target_doc_ids = list(target_doc_ids)
+    updated = DocumentService.batch_update_metadata(kb_id, target_doc_ids, updates, deletes)
+    return get_json_result(data={"updated": updated, "matched_docs": len(target_doc_ids)})
+
+
+@manager.route("/update_metadata_setting", methods=["POST"])  # noqa: F821
+@login_required
+@validate_request("doc_id", "metadata")
+async def update_metadata_setting():
+    req = await get_request_json()
+    if not DocumentService.accessible(req["doc_id"], current_user.id):
+        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
+
+    e, doc = DocumentService.get_by_id(req["doc_id"])
+    if not e:
+        return get_data_error_result(message="Document not found!")
+
+    DocumentService.update_parser_config(doc.id, {"metadata": req["metadata"]})
+    e, doc = DocumentService.get_by_id(doc.id)
+    if not e:
+        return get_data_error_result(message="Document not found!")
+
+    return get_json_result(data=doc.to_dict())
+
+
 @manager.route("/thumbnails", methods=["GET"])  # noqa: F821
 # @login_required
 def thumbnails():
@@ -362,7 +533,7 @@ async def change_status():
                 continue
             e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
             if not e:
-                result[doc_id] = {"error": "Can't find this knowledgebase!"}
+                result[doc_id] = {"error": "Can't find this dataset!"}
                 continue
             if not DocumentService.update_by_id(doc_id, {"status": str(status)}):
                 result[doc_id] = {"error": "Database error (Document update)!"}
@@ -436,10 +607,17 @@ async def run():
                 DocumentService.update_by_id(id, info)
                 if req.get("delete", False):
                     TaskService.filter_delete([Task.doc_id == id])
-                    if settings.docStoreConn.indexExist(search.index_name(tenant_id), doc.kb_id):
+                    if settings.docStoreConn.index_exist(search.index_name(tenant_id), doc.kb_id):
                         settings.docStoreConn.delete({"doc_id": id}, search.index_name(tenant_id), doc.kb_id)
 
                 if str(req["run"]) == TaskStatus.RUNNING.value:
+                    if req.get("apply_kb"):
+                        e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
+                        if not e:
+                            raise LookupError("Can't find this dataset!")
+                        doc.parser_config["enable_metadata"] = kb.parser_config.get("enable_metadata", False)
+                        doc.parser_config["metadata"] = kb.parser_config.get("metadata", {})
+                        DocumentService.update_parser_config(doc.id, doc.parser_config)
                     doc_dict = doc.to_dict()
                     DocumentService.run(tenant_id, doc_dict, kb_table_num_map)
 
@@ -470,7 +648,7 @@ async def rename():
 
             for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
                 if d.name == req["name"]:
-                    return get_data_error_result(message="Duplicated document name in the same knowledgebase.")
+                    return get_data_error_result(message="Duplicated document name in the same dataset.")
 
             if not DocumentService.update_by_id(req["doc_id"], {"name": req["name"]}):
                 return get_data_error_result(message="Database error (Document rename)!")
@@ -487,7 +665,7 @@ async def rename():
                 "title_tks": title_tks,
                 "title_sm_tks": rag_tokenizer.fine_grained_tokenize(title_tks),
             }
-            if settings.docStoreConn.indexExist(search.index_name(tenant_id), doc.kb_id):
+            if settings.docStoreConn.index_exist(search.index_name(tenant_id), doc.kb_id):
                 settings.docStoreConn.update(
                     {"doc_id": req["doc_id"]},
                     es_body,
@@ -568,7 +746,7 @@ async def change_parser():
             tenant_id = DocumentService.get_tenant_id(req["doc_id"])
             if not tenant_id:
                 return get_data_error_result(message="Tenant not found!")
-            if settings.docStoreConn.indexExist(search.index_name(tenant_id), doc.kb_id):
+            if settings.docStoreConn.index_exist(search.index_name(tenant_id), doc.kb_id):
                 settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id)
         return None
 
@@ -698,7 +876,10 @@ async def set_meta():
         if not isinstance(meta, dict):
             return get_json_result(data=False, message="Only dictionary type supported.", code=RetCode.ARGUMENT_ERROR)
         for k, v in meta.items():
-            if not isinstance(v, str) and not isinstance(v, int) and not isinstance(v, float):
+            if isinstance(v, list):
+                if not all(isinstance(i, (str, int, float)) for i in v):
+                    return get_json_result(data=False, message=f"The type is not supported in list: {v}", code=RetCode.ARGUMENT_ERROR)
+            elif not isinstance(v, (str, int, float)):
                 return get_json_result(data=False, message=f"The type is not supported: {v}", code=RetCode.ARGUMENT_ERROR)
     except Exception as e:
         return get_json_result(data=False, message=f"Json syntax error: {e}", code=RetCode.ARGUMENT_ERROR)

api/apps/file2document_app.py

@@ -68,7 +68,7 @@ async def convert():
                     e, kb = KnowledgebaseService.get_by_id(kb_id)
                     if not e:
                         return get_data_error_result(
-                            message="Can't find this knowledgebase!")
+                            message="Can't find this dataset!")
                     e, file = FileService.get_by_id(id)
                     if not e:
                         return get_data_error_result(

api/apps/kb_app.py

@@ -39,9 +39,9 @@ from api.utils.api_utils import get_json_result
 from rag.nlp import search
 from api.constants import DATASET_NAME_LIMIT
 from rag.utils.redis_conn import REDIS_CONN
-from rag.utils.doc_store_conn import OrderByExpr
 from common.constants import RetCode, PipelineTaskType, StatusEnum, VALID_TASK_STATUS, FileSource, LLMType, PAGERANK_FLD
 from common import settings
+from common.doc_store.doc_store_base import OrderByExpr
 from api.apps import login_required, current_user
 
 
@@ -93,19 +93,32 @@ async def update():
         if not KnowledgebaseService.query(
                 created_by=current_user.id, id=req["kb_id"]):
             return get_json_result(
-                data=False, message='Only owner of knowledgebase authorized for this operation.',
+                data=False, message='Only owner of dataset authorized for this operation.',
                 code=RetCode.OPERATING_ERROR)
 
         e, kb = KnowledgebaseService.get_by_id(req["kb_id"])
+
+        # Rename folder in FileService
+        if e and req["name"].lower() != kb.name.lower():
+            FileService.filter_update(
+                [
+                    File.tenant_id == kb.tenant_id,
+                    File.source_type == FileSource.KNOWLEDGEBASE,
+                    File.type == "folder",
+                    File.name == kb.name,
+                ],
+                {"name": req["name"]},
+            )
+
         if not e:
             return get_data_error_result(
-                message="Can't find this knowledgebase!")
+                message="Can't find this dataset!")
 
         if req["name"].lower() != kb.name.lower() \
                 and len(
             KnowledgebaseService.query(name=req["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) >= 1:
             return get_data_error_result(
-                message="Duplicated knowledgebase name.")
+                message="Duplicated dataset name.")
 
         del req["kb_id"]
         connectors = []
@@ -150,6 +163,21 @@ async def update():
         return server_error_response(e)
 
 
+@manager.route('/update_metadata_setting', methods=['post'])  # noqa: F821
+@login_required
+@validate_request("kb_id", "metadata")
+async def update_metadata_setting():
+    req = await get_request_json()
+    e, kb = KnowledgebaseService.get_by_id(req["kb_id"])
+    if not e:
+        return get_data_error_result(
+            message="Database error (Knowledgebase rename)!")
+    kb = kb.to_dict()
+    kb["parser_config"]["metadata"] = req["metadata"]
+    KnowledgebaseService.update_by_id(kb["id"], kb)
+    return get_json_result(data=kb)
+
+
 @manager.route('/detail', methods=['GET'])  # noqa: F821
 @login_required
 def detail():
@@ -162,12 +190,12 @@ def detail():
                 break
         else:
             return get_json_result(
-                data=False, message='Only owner of knowledgebase authorized for this operation.',
+                data=False, message='Only owner of dataset authorized for this operation.',
                 code=RetCode.OPERATING_ERROR)
         kb = KnowledgebaseService.get_detail(kb_id)
         if not kb:
             return get_data_error_result(
-                message="Can't find this knowledgebase!")
+                message="Can't find this dataset!")
         kb["size"] = DocumentService.get_total_size_by_kb_id(kb_id=kb["id"],keywords="", run_status=[], types=[])
         kb["connectors"] = Connector2KbService.list_connectors(kb_id)
 
@@ -232,7 +260,7 @@ async def rm():
             created_by=current_user.id, id=req["kb_id"])
         if not kbs:
             return get_json_result(
-                data=False, message='Only owner of knowledgebase authorized for this operation.',
+                data=False, message='Only owner of dataset authorized for this operation.',
                 code=RetCode.OPERATING_ERROR)
 
         def _rm_sync():
@@ -245,13 +273,19 @@ async def rm():
                     FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
                 File2DocumentService.delete_by_document_id(doc.id)
             FileService.filter_delete(
-                [File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
+                [
+                    File.tenant_id == kbs[0].tenant_id,
+                    File.source_type == FileSource.KNOWLEDGEBASE,
+                    File.type == "folder",
+                    File.name == kbs[0].name,
+                ]
+            )
             if not KnowledgebaseService.delete_by_id(req["kb_id"]):
                 return get_data_error_result(
                     message="Database error (Knowledgebase removal)!")
             for kb in kbs:
                 settings.docStoreConn.delete({"kb_id": kb.id}, search.index_name(kb.tenant_id), kb.id)
-                settings.docStoreConn.deleteIdx(search.index_name(kb.tenant_id), kb.id)
+                settings.docStoreConn.delete_idx(search.index_name(kb.tenant_id), kb.id)
                 if hasattr(settings.STORAGE_IMPL, 'remove_bucket'):
                     settings.STORAGE_IMPL.remove_bucket(kb.id)
             return get_json_result(data=True)
@@ -352,7 +386,7 @@ def knowledge_graph(kb_id):
     }
 
     obj = {"graph": {}, "mind_map": {}}
-    if not settings.docStoreConn.indexExist(search.index_name(kb.tenant_id), kb_id):
+    if not settings.docStoreConn.index_exist(search.index_name(kb.tenant_id), kb_id):
         return get_json_result(data=obj)
     sres = settings.retriever.search(req, search.index_name(kb.tenant_id), [kb_id])
     if not len(sres.ids):
@@ -824,11 +858,11 @@ async def check_embedding():
         index_nm = search.index_name(tenant_id)
 
         res0 = docStoreConn.search(
-            selectFields=[], highlightFields=[],
+            select_fields=[], highlight_fields=[],
             condition={"kb_id": kb_id, "available_int": 1},
-            matchExprs=[], orderBy=OrderByExpr(),
+            match_expressions=[], order_by=OrderByExpr(),
             offset=0, limit=1,
-            indexNames=index_nm, knowledgebaseIds=[kb_id]
+            index_names=index_nm, knowledgebase_ids=[kb_id]
         )
         total = docStoreConn.get_total(res0)
         if total <= 0:
@@ -840,14 +874,14 @@ async def check_embedding():
 
         for off in offsets:
             res1 = docStoreConn.search(
-                selectFields=list(base_fields),
-                highlightFields=[],
+                select_fields=list(base_fields),
+                highlight_fields=[],
                 condition={"kb_id": kb_id, "available_int": 1},
-                matchExprs=[], orderBy=OrderByExpr(),
+                match_expressions=[], order_by=OrderByExpr(),
                 offset=off, limit=1,
-                indexNames=index_nm, knowledgebaseIds=[kb_id]
+                index_names=index_nm, knowledgebase_ids=[kb_id]
             )
-            ids = docStoreConn.get_chunk_ids(res1)
+            ids = docStoreConn.get_doc_ids(res1)
             if not ids:
                 continue
 

api/apps/langfuse_app.py

@@ -34,8 +34,9 @@ async def set_api_key():
     if not all([secret_key, public_key, host]):
         return get_error_data_result(message="Missing required fields")
 
+    current_user_id = current_user.id
     langfuse_keys = dict(
-        tenant_id=current_user.id,
+        tenant_id=current_user_id,
         secret_key=secret_key,
         public_key=public_key,
         host=host,
@@ -45,23 +46,24 @@ async def set_api_key():
     if not langfuse.auth_check():
         return get_error_data_result(message="Invalid Langfuse keys")
 
-    langfuse_entry = TenantLangfuseService.filter_by_tenant(tenant_id=current_user.id)
+    langfuse_entry = TenantLangfuseService.filter_by_tenant(tenant_id=current_user_id)
     with DB.atomic():
         try:
             if not langfuse_entry:
                 TenantLangfuseService.save(**langfuse_keys)
             else:
-                TenantLangfuseService.update_by_tenant(tenant_id=current_user.id, langfuse_keys=langfuse_keys)
+                TenantLangfuseService.update_by_tenant(tenant_id=current_user_id, langfuse_keys=langfuse_keys)
             return get_json_result(data=langfuse_keys)
         except Exception as e:
-            server_error_response(e)
+            return server_error_response(e)
 
 
 @manager.route("/api_key", methods=["GET"])  # noqa: F821
 @login_required
 @validate_request()
 def get_api_key():
-    langfuse_entry = TenantLangfuseService.filter_by_tenant_with_info(tenant_id=current_user.id)
+    current_user_id = current_user.id
+    langfuse_entry = TenantLangfuseService.filter_by_tenant_with_info(tenant_id=current_user_id)
     if not langfuse_entry:
         return get_json_result(message="Have not record any Langfuse keys.")
 
@@ -72,7 +74,7 @@ def get_api_key():
     except langfuse.api.core.api_error.ApiError as api_err:
         return get_json_result(message=f"Error from Langfuse: {api_err}")
     except Exception as e:
-        server_error_response(e)
+        return server_error_response(e)
 
     langfuse_entry["project_id"] = langfuse.api.projects.get().dict()["data"][0]["id"]
     langfuse_entry["project_name"] = langfuse.api.projects.get().dict()["data"][0]["name"]
@@ -84,7 +86,8 @@ def get_api_key():
 @login_required
 @validate_request()
 def delete_api_key():
-    langfuse_entry = TenantLangfuseService.filter_by_tenant(tenant_id=current_user.id)
+    current_user_id = current_user.id
+    langfuse_entry = TenantLangfuseService.filter_by_tenant(tenant_id=current_user_id)
     if not langfuse_entry:
         return get_json_result(message="Have not record any Langfuse keys.")
 
@@ -93,4 +96,4 @@ def delete_api_key():
             TenantLangfuseService.delete_model(langfuse_entry)
             return get_json_result(data=True)
         except Exception as e:
-            server_error_response(e)
+            return server_error_response(e)

api/apps/llm_app.py

@@ -25,7 +25,7 @@ from api.utils.api_utils import get_allowed_llm_factories, get_data_error_result
 from common.constants import StatusEnum, LLMType
 from api.db.db_models import TenantLLM
 from rag.utils.base64_image import test_image
-from rag.llm import EmbeddingModel, ChatModel, RerankModel, CvModel, TTSModel
+from rag.llm import EmbeddingModel, ChatModel, RerankModel, CvModel, TTSModel, OcrModel, Seq2txtModel
 
 
 @manager.route("/factories", methods=["GET"])  # noqa: F821
@@ -43,7 +43,13 @@ def factories():
                 mdl_types[m.fid] = set([])
             mdl_types[m.fid].add(m.model_type)
         for f in fac:
-            f["model_types"] = list(mdl_types.get(f["name"], [LLMType.CHAT, LLMType.EMBEDDING, LLMType.RERANK, LLMType.IMAGE2TEXT, LLMType.SPEECH2TEXT, LLMType.TTS]))
+            f["model_types"] = list(
+                mdl_types.get(
+                    f["name"],
+                    [LLMType.CHAT, LLMType.EMBEDDING, LLMType.RERANK, LLMType.IMAGE2TEXT, LLMType.SPEECH2TEXT, LLMType.TTS, LLMType.OCR],
+                )
+            )
+
         return get_json_result(data=fac)
     except Exception as e:
         return server_error_response(e)
@@ -74,7 +80,7 @@ async def set_api_key():
             assert factory in ChatModel, f"Chat model from {factory} is not supported yet."
             mdl = ChatModel[factory](req["api_key"], llm.llm_name, base_url=req.get("base_url"), **extra)
             try:
-                m, tc = mdl.chat(None, [{"role": "user", "content": "Hello! How are you doing!"}], {"temperature": 0.9, "max_tokens": 50})
+                m, tc = await mdl.async_chat(None, [{"role": "user", "content": "Hello! How are you doing!"}], {"temperature": 0.9, "max_tokens": 50})
                 if m.find("**ERROR**") >= 0:
                     raise Exception(m)
                 chat_passed = True
@@ -151,7 +157,7 @@ async def add_llm():
     elif factory == "Bedrock":
         # For Bedrock, due to its special authentication method
         # Assemble bedrock_ak, bedrock_sk, bedrock_region
-        api_key = apikey_json(["bedrock_ak", "bedrock_sk", "bedrock_region"])
+        api_key = apikey_json(["auth_mode", "bedrock_ak", "bedrock_sk", "bedrock_region", "aws_role_arn"])
 
     elif factory == "LocalAI":
         llm_name += "___LocalAI"
@@ -186,6 +192,9 @@ async def add_llm():
     elif factory == "OpenRouter":
         api_key = apikey_json(["api_key", "provider_order"])
 
+    elif factory == "MinerU":
+        api_key = apikey_json(["api_key", "provider_order"])
+
     llm = {
         "tenant_id": current_user.id,
         "llm_factory": factory,
@@ -199,61 +208,83 @@ async def add_llm():
     msg = ""
     mdl_nm = llm["llm_name"].split("___")[0]
     extra = {"provider": factory}
-    if llm["model_type"] == LLMType.EMBEDDING.value:
-        assert factory in EmbeddingModel, f"Embedding model from {factory} is not supported yet."
-        mdl = EmbeddingModel[factory](key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"])
-        try:
-            arr, tc = mdl.encode(["Test if the api key is available"])
-            if len(arr[0]) == 0:
-                raise Exception("Fail")
-        except Exception as e:
-            msg += f"\nFail to access embedding model({mdl_nm})." + str(e)
-    elif llm["model_type"] == LLMType.CHAT.value:
-        assert factory in ChatModel, f"Chat model from {factory} is not supported yet."
-        mdl = ChatModel[factory](
-            key=llm["api_key"],
-            model_name=mdl_nm,
-            base_url=llm["api_base"],
-            **extra,
-        )
-        try:
-            m, tc = mdl.chat(None, [{"role": "user", "content": "Hello! How are you doing!"}], {"temperature": 0.9})
-            if not tc and m.find("**ERROR**:") >= 0:
-                raise Exception(m)
-        except Exception as e:
-            msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
-    elif llm["model_type"] == LLMType.RERANK:
-        assert factory in RerankModel, f"RE-rank model from {factory} is not supported yet."
-        try:
-            mdl = RerankModel[factory](key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"])
-            arr, tc = mdl.similarity("Hello~ RAGFlower!", ["Hi, there!", "Ohh, my friend!"])
-            if len(arr) == 0:
-                raise Exception("Not known.")
-        except KeyError:
-            msg += f"{factory} dose not support this model({factory}/{mdl_nm})"
-        except Exception as e:
-            msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
-    elif llm["model_type"] == LLMType.IMAGE2TEXT.value:
-        assert factory in CvModel, f"Image to text model from {factory} is not supported yet."
-        mdl = CvModel[factory](key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"])
-        try:
-            image_data = test_image
-            m, tc = mdl.describe(image_data)
-            if not tc and m.find("**ERROR**:") >= 0:
-                raise Exception(m)
-        except Exception as e:
-            msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
-    elif llm["model_type"] == LLMType.TTS:
-        assert factory in TTSModel, f"TTS model from {factory} is not supported yet."
-        mdl = TTSModel[factory](key=llm["api_key"], model_name=mdl_nm, base_url=llm["api_base"])
-        try:
-            for resp in mdl.tts("Hello~ RAGFlower!"):
-                pass
-        except RuntimeError as e:
-            msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
-    else:
-        # TODO: check other type of models
-        pass
+    model_type = llm["model_type"]
+    model_api_key = llm["api_key"]
+    model_base_url = llm.get("api_base", "")
+    match model_type:
+        case LLMType.EMBEDDING.value:
+            assert factory in EmbeddingModel, f"Embedding model from {factory} is not supported yet."
+            mdl = EmbeddingModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
+            try:
+                arr, tc = mdl.encode(["Test if the api key is available"])
+                if len(arr[0]) == 0:
+                    raise Exception("Fail")
+            except Exception as e:
+                msg += f"\nFail to access embedding model({mdl_nm})." + str(e)
+        case LLMType.CHAT.value:
+            assert factory in ChatModel, f"Chat model from {factory} is not supported yet."
+            mdl = ChatModel[factory](
+                key=model_api_key,
+                model_name=mdl_nm,
+                base_url=model_base_url,
+                **extra,
+            )
+            try:
+                m, tc = await mdl.async_chat(None, [{"role": "user", "content": "Hello! How are you doing!"}],
+                                             {"temperature": 0.9})
+                if not tc and m.find("**ERROR**:") >= 0:
+                    raise Exception(m)
+            except Exception as e:
+                msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
+
+        case LLMType.RERANK.value:
+            assert factory in RerankModel, f"RE-rank model from {factory} is not supported yet."
+            try:
+                mdl = RerankModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
+                arr, tc = mdl.similarity("Hello~ RAGFlower!", ["Hi, there!", "Ohh, my friend!"])
+                if len(arr) == 0:
+                    raise Exception("Not known.")
+            except KeyError:
+                msg += f"{factory} dose not support this model({factory}/{mdl_nm})"
+            except Exception as e:
+                msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
+
+        case LLMType.IMAGE2TEXT.value:
+            assert factory in CvModel, f"Image to text model from {factory} is not supported yet."
+            mdl = CvModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
+            try:
+                image_data = test_image
+                m, tc = mdl.describe(image_data)
+                if not tc and m.find("**ERROR**:") >= 0:
+                    raise Exception(m)
+            except Exception as e:
+                msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
+        case LLMType.TTS.value:
+            assert factory in TTSModel, f"TTS model from {factory} is not supported yet."
+            mdl = TTSModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
+            try:
+                for resp in mdl.tts("Hello~ RAGFlower!"):
+                    pass
+            except RuntimeError as e:
+                msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
+        case LLMType.OCR.value:
+            assert factory in OcrModel, f"OCR model from {factory} is not supported yet."
+            try:
+                mdl = OcrModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
+                ok, reason = mdl.check_available()
+                if not ok:
+                    raise RuntimeError(reason or "Model not available")
+            except Exception as e:
+                msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
+        case LLMType.SPEECH2TEXT:
+            assert factory in Seq2txtModel, f"Speech model from {factory} is not supported yet."
+            try:
+                mdl = Seq2txtModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
+                # TODO: check the availability
+            except Exception as e:
+                msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
+        case _:
+            raise RuntimeError(f"Unknown model type: {model_type}")
 
     if msg:
         return get_data_error_result(message=msg)
@@ -297,6 +328,7 @@ async def delete_factory():
 @login_required
 def my_llms():
     try:
+        TenantLLMService.ensure_mineru_from_env(current_user.id)
         include_details = request.args.get("include_details", "false").lower() == "true"
 
         if include_details:
@@ -344,6 +376,7 @@ def list_app():
     weighted = []
     model_type = request.args.get("model_type")
     try:
+        TenantLLMService.ensure_mineru_from_env(current_user.id)
         objs = TenantLLMService.query(tenant_id=current_user.id)
         facts = set([o.to_dict()["llm_factory"] for o in objs if o.api_key and o.status == StatusEnum.VALID.value])
         status = {(o.llm_name + "@" + o.llm_factory) for o in objs if o.status == StatusEnum.VALID.value}

api/apps/mcp_server_app.py

@@ -13,6 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import asyncio
+
 from quart import Response, request
 from api.apps import current_user, login_required
 
@@ -106,7 +108,7 @@ async def create() -> Response:
             return get_data_error_result(message="Tenant not found.")
 
         mcp_server = MCPServer(id=server_name, name=server_name, url=url, server_type=server_type, variables=variables, headers=headers)
-        server_tools, err_message = get_mcp_tools([mcp_server], timeout)
+        server_tools, err_message = await asyncio.to_thread(get_mcp_tools, [mcp_server], timeout)
         if err_message:
             return get_data_error_result(err_message)
 
@@ -158,7 +160,7 @@ async def update() -> Response:
         req["id"] = mcp_id
 
         mcp_server = MCPServer(id=server_name, name=server_name, url=url, server_type=server_type, variables=variables, headers=headers)
-        server_tools, err_message = get_mcp_tools([mcp_server], timeout)
+        server_tools, err_message = await asyncio.to_thread(get_mcp_tools, [mcp_server], timeout)
         if err_message:
             return get_data_error_result(err_message)
 
@@ -242,7 +244,7 @@ async def import_multiple() -> Response:
             headers = {"authorization_token": config["authorization_token"]} if "authorization_token" in config else {}
             variables = {k: v for k, v in config.items() if k not in {"type", "url", "headers"}}
             mcp_server = MCPServer(id=new_name, name=new_name, url=config["url"], server_type=config["type"], variables=variables, headers=headers)
-            server_tools, err_message = get_mcp_tools([mcp_server], timeout)
+            server_tools, err_message = await asyncio.to_thread(get_mcp_tools, [mcp_server], timeout)
             if err_message:
                 results.append({"server": base_name, "success": False, "message": err_message})
                 continue
@@ -322,9 +324,8 @@ async def list_tools() -> Response:
                 tool_call_sessions.append(tool_call_session)
 
                 try:
-                    tools = tool_call_session.get_tools(timeout)
+                    tools = await asyncio.to_thread(tool_call_session.get_tools, timeout)
                 except Exception as e:
-                    tools = []
                     return get_data_error_result(message=f"MCP list tools error: {e}")
 
                 results[server_key] = []
@@ -340,7 +341,7 @@ async def list_tools() -> Response:
         return server_error_response(e)
     finally:
         # PERF: blocking call to close sessions — consider moving to background thread or task queue
-        close_multiple_mcp_toolcall_sessions(tool_call_sessions)
+        await asyncio.to_thread(close_multiple_mcp_toolcall_sessions, tool_call_sessions)
 
 
 @manager.route("/test_tool", methods=["POST"])  # noqa: F821
@@ -367,10 +368,10 @@ async def test_tool() -> Response:
 
         tool_call_session = MCPToolCallSession(mcp_server, mcp_server.variables)
         tool_call_sessions.append(tool_call_session)
-        result = tool_call_session.tool_call(tool_name, arguments, timeout)
+        result = await asyncio.to_thread(tool_call_session.tool_call, tool_name, arguments, timeout)
 
         # PERF: blocking call to close sessions — consider moving to background thread or task queue
-        close_multiple_mcp_toolcall_sessions(tool_call_sessions)
+        await asyncio.to_thread(close_multiple_mcp_toolcall_sessions, tool_call_sessions)
         return get_json_result(data=result)
     except Exception as e:
         return server_error_response(e)
@@ -424,13 +425,12 @@ async def test_mcp() -> Response:
         tool_call_session = MCPToolCallSession(mcp_server, mcp_server.variables)
 
         try:
-            tools = tool_call_session.get_tools(timeout)
+            tools = await asyncio.to_thread(tool_call_session.get_tools, timeout)
         except Exception as e:
-            tools = []
             return get_data_error_result(message=f"Test MCP error: {e}")
         finally:
             # PERF: blocking call to close sessions — consider moving to background thread or task queue
-            close_multiple_mcp_toolcall_sessions([tool_call_session])
+            await asyncio.to_thread(close_multiple_mcp_toolcall_sessions, [tool_call_session])
 
         for tool in tools:
             tool_dict = tool.model_dump()

api/apps/memories_app.py

@@ -0,0 +1,227 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import logging
+
+from quart import request
+from api.apps import login_required, current_user
+from api.db import TenantPermission
+from api.db.services.memory_service import MemoryService
+from api.db.services.user_service import UserTenantService
+from api.db.services.canvas_service import UserCanvasService
+from api.db.joint_services.memory_message_service import get_memory_size_cache, judge_system_prompt_is_default
+from api.utils.api_utils import validate_request, get_request_json, get_error_argument_result, get_json_result
+from api.utils.memory_utils import format_ret_data_from_memory, get_memory_type_human
+from api.constants import MEMORY_NAME_LIMIT, MEMORY_SIZE_LIMIT
+from memory.services.messages import MessageService
+from memory.utils.prompt_util import PromptAssembler
+from common.constants import MemoryType, RetCode, ForgettingPolicy
+
+
+@manager.route("", methods=["POST"])  # noqa: F821
+@login_required
+@validate_request("name", "memory_type", "embd_id", "llm_id")
+async def create_memory():
+    req = await get_request_json()
+    # check name length
+    name = req["name"]
+    memory_name = name.strip()
+    if len(memory_name) == 0:
+        return get_error_argument_result("Memory name cannot be empty or whitespace.")
+    if len(memory_name) > MEMORY_NAME_LIMIT:
+        return get_error_argument_result(f"Memory name '{memory_name}' exceeds limit of {MEMORY_NAME_LIMIT}.")
+    # check memory_type valid
+    memory_type = set(req["memory_type"])
+    invalid_type = memory_type - {e.name.lower() for e in MemoryType}
+    if invalid_type:
+        return get_error_argument_result(f"Memory type '{invalid_type}' is not supported.")
+    memory_type = list(memory_type)
+
+    try:
+        res, memory = MemoryService.create_memory(
+            tenant_id=current_user.id,
+            name=memory_name,
+            memory_type=memory_type,
+            embd_id=req["embd_id"],
+            llm_id=req["llm_id"]
+        )
+
+        if res:
+            return get_json_result(message=True, data=format_ret_data_from_memory(memory))
+        else:
+            return get_json_result(message=memory, code=RetCode.SERVER_ERROR)
+
+    except Exception as e:
+        return get_json_result(message=str(e), code=RetCode.SERVER_ERROR)
+
+
+@manager.route("/<memory_id>", methods=["PUT"])  # noqa: F821
+@login_required
+async def update_memory(memory_id):
+    req = await get_request_json()
+    update_dict = {}
+    # check name length
+    if "name" in req:
+        name = req["name"]
+        memory_name = name.strip()
+        if len(memory_name) == 0:
+            return get_error_argument_result("Memory name cannot be empty or whitespace.")
+        if len(memory_name) > MEMORY_NAME_LIMIT:
+            return get_error_argument_result(f"Memory name '{memory_name}' exceeds limit of {MEMORY_NAME_LIMIT}.")
+        update_dict["name"] = memory_name
+    # check permissions valid
+    if req.get("permissions"):
+        if req["permissions"] not in [e.value for e in TenantPermission]:
+            return get_error_argument_result(f"Unknown permission '{req['permissions']}'.")
+        update_dict["permissions"] = req["permissions"]
+    if req.get("llm_id"):
+        update_dict["llm_id"] = req["llm_id"]
+    if req.get("embd_id"):
+        update_dict["embd_id"] = req["embd_id"]
+    if req.get("memory_type"):
+        memory_type = set(req["memory_type"])
+        invalid_type = memory_type - {e.name.lower() for e in MemoryType}
+        if invalid_type:
+            return get_error_argument_result(f"Memory type '{invalid_type}' is not supported.")
+        update_dict["memory_type"] = list(memory_type)
+    # check memory_size valid
+    if req.get("memory_size"):
+        if not 0 < int(req["memory_size"]) <= MEMORY_SIZE_LIMIT:
+            return get_error_argument_result(f"Memory size should be in range (0, {MEMORY_SIZE_LIMIT}] Bytes.")
+        update_dict["memory_size"] = req["memory_size"]
+    # check forgetting_policy valid
+    if req.get("forgetting_policy"):
+        if req["forgetting_policy"] not in [e.value for e in ForgettingPolicy]:
+            return get_error_argument_result(f"Forgetting policy '{req['forgetting_policy']}' is not supported.")
+        update_dict["forgetting_policy"] = req["forgetting_policy"]
+    # check temperature valid
+    if "temperature" in req:
+        temperature = float(req["temperature"])
+        if not 0 <= temperature <= 1:
+            return get_error_argument_result("Temperature should be in range [0, 1].")
+        update_dict["temperature"] = temperature
+    # allow update to empty fields
+    for field in ["avatar", "description", "system_prompt", "user_prompt"]:
+        if field in req:
+            update_dict[field] = req[field]
+    current_memory = MemoryService.get_by_memory_id(memory_id)
+    if not current_memory:
+        return get_json_result(code=RetCode.NOT_FOUND, message=f"Memory '{memory_id}' not found.")
+
+    memory_dict = current_memory.to_dict()
+    memory_dict.update({"memory_type": get_memory_type_human(current_memory.memory_type)})
+    to_update = {}
+    for k, v in update_dict.items():
+        if isinstance(v, list) and set(memory_dict[k]) != set(v):
+            to_update[k] = v
+        elif memory_dict[k] != v:
+            to_update[k] = v
+
+    if not to_update:
+        return get_json_result(message=True, data=memory_dict)
+    # check memory empty when update embd_id, memory_type
+    memory_size = get_memory_size_cache(memory_id, current_memory.tenant_id)
+    not_allowed_update = [f for f in ["embd_id", "memory_type"] if f in to_update and memory_size > 0]
+    if not_allowed_update:
+        return get_error_argument_result(f"Can't update {not_allowed_update} when memory isn't empty.")
+    if "memory_type" in to_update:
+        if "system_prompt" not in to_update and judge_system_prompt_is_default(current_memory.system_prompt, current_memory.memory_type):
+            # update old default prompt, assemble a new one
+            to_update["system_prompt"] = PromptAssembler.assemble_system_prompt({"memory_type": to_update["memory_type"]})
+
+    try:
+        MemoryService.update_memory(current_memory.tenant_id, memory_id, to_update)
+        updated_memory = MemoryService.get_by_memory_id(memory_id)
+        return get_json_result(message=True, data=format_ret_data_from_memory(updated_memory))
+
+    except Exception as e:
+        logging.error(e)
+        return get_json_result(message=str(e), code=RetCode.SERVER_ERROR)
+
+
+@manager.route("/<memory_id>", methods=["DELETE"])  # noqa: F821
+@login_required
+async def delete_memory(memory_id):
+    memory = MemoryService.get_by_memory_id(memory_id)
+    if not memory:
+        return get_json_result(message=True, code=RetCode.NOT_FOUND)
+    try:
+        MemoryService.delete_memory(memory_id)
+        MessageService.delete_message({"memory_id": memory_id}, memory.tenant_id, memory_id)
+        return get_json_result(message=True)
+    except Exception as e:
+        logging.error(e)
+        return get_json_result(message=str(e), code=RetCode.SERVER_ERROR)
+
+
+@manager.route("", methods=["GET"])  # noqa: F821
+@login_required
+async def list_memory():
+    args = request.args
+    try:
+        tenant_ids = args.getlist("tenant_id")
+        memory_types = args.getlist("memory_type")
+        storage_type = args.get("storage_type")
+        keywords = args.get("keywords", "")
+        page = int(args.get("page", 1))
+        page_size = int(args.get("page_size", 50))
+        # make filter dict
+        filter_dict = {"memory_type": memory_types, "storage_type": storage_type}
+        if not tenant_ids:
+            # restrict to current user's tenants
+            user_tenants = UserTenantService.get_user_tenant_relation_by_user_id(current_user.id)
+            filter_dict["tenant_id"] = [tenant["tenant_id"] for tenant in user_tenants]
+        else:
+            filter_dict["tenant_id"] = tenant_ids
+
+        memory_list, count = MemoryService.get_by_filter(filter_dict, keywords, page, page_size)
+        [memory.update({"memory_type": get_memory_type_human(memory["memory_type"])}) for memory in memory_list]
+        return get_json_result(message=True, data={"memory_list": memory_list, "total_count": count})
+
+    except Exception as e:
+        logging.error(e)
+        return get_json_result(message=str(e), code=RetCode.SERVER_ERROR)
+
+
+@manager.route("/<memory_id>/config", methods=["GET"])  # noqa: F821
+@login_required
+async def get_memory_config(memory_id):
+    memory = MemoryService.get_with_owner_name_by_id(memory_id)
+    if not memory:
+        return get_json_result(code=RetCode.NOT_FOUND, message=f"Memory '{memory_id}' not found.")
+    return get_json_result(message=True, data=format_ret_data_from_memory(memory))
+
+
+@manager.route("/<memory_id>", methods=["GET"])  # noqa: F821
+@login_required
+async def get_memory_detail(memory_id):
+    args = request.args
+    agent_ids = args.getlist("agent_id")
+    keywords = args.get("keywords", "")
+    keywords = keywords.strip()
+    page = int(args.get("page", 1))
+    page_size = int(args.get("page_size", 50))
+    memory = MemoryService.get_by_memory_id(memory_id)
+    if not memory:
+        return get_json_result(code=RetCode.NOT_FOUND, message=f"Memory '{memory_id}' not found.")
+    messages = MessageService.list_message(
+        memory.tenant_id, memory_id, agent_ids, keywords, page, page_size)
+    agent_name_mapping = {}
+    if messages["message_list"]:
+        agent_list = UserCanvasService.get_basic_info_by_canvas_ids([message["agent_id"] for message in messages["message_list"]])
+        agent_name_mapping = {agent["id"]: agent["title"] for agent in agent_list}
+    for message in messages["message_list"]:
+        message["agent_name"] = agent_name_mapping.get(message["agent_id"], "Unknown")
+    return get_json_result(data={"messages": messages, "storage_type": memory.storage_type}, message=True)

api/apps/messages_app.py

@@ -0,0 +1,168 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from quart import request
+from api.apps import login_required
+from api.db.services.memory_service import MemoryService
+from common.time_utils import current_timestamp, timestamp_to_date
+
+from memory.services.messages import MessageService
+from api.db.joint_services import memory_message_service
+from api.utils.api_utils import validate_request, get_request_json, get_error_argument_result, get_json_result
+from common.constants import RetCode
+
+
+@manager.route("", methods=["POST"]) # noqa: F821
+@login_required
+@validate_request("memory_id", "agent_id", "session_id", "user_input", "agent_response")
+async def add_message():
+
+    req = await get_request_json()
+    memory_ids = req["memory_id"]
+    agent_id = req["agent_id"]
+    session_id = req["session_id"]
+    user_id = req["user_id"] if req.get("user_id") else ""
+    user_input = req["user_input"]
+    agent_response = req["agent_response"]
+
+    res = []
+    for memory_id in memory_ids:
+        success, msg = await memory_message_service.save_to_memory(
+            memory_id,
+            {
+                "user_id": user_id,
+                "agent_id": agent_id,
+                "session_id": session_id,
+                "user_input": user_input,
+                "agent_response": agent_response
+            }
+        )
+        res.append({
+            "memory_id": memory_id,
+            "success": success,
+            "message": msg
+        })
+
+    if all([r["success"] for r in res]):
+        return get_json_result(message="Successfully added to memories.")
+
+    return get_json_result(code=RetCode.SERVER_ERROR, message="Some messages failed to add.", data=res)
+
+
+@manager.route("/<memory_id>:<message_id>", methods=["DELETE"]) # noqa: F821
+@login_required
+async def forget_message(memory_id: str, message_id: int):
+
+    memory = MemoryService.get_by_memory_id(memory_id)
+    if not memory:
+        return get_json_result(code=RetCode.NOT_FOUND, message=f"Memory '{memory_id}' not found.")
+
+    forget_time = timestamp_to_date(current_timestamp())
+    update_succeed = MessageService.update_message(
+        {"memory_id": memory_id, "message_id": int(message_id)},
+        {"forget_at": forget_time},
+        memory.tenant_id, memory_id)
+    if update_succeed:
+        return get_json_result(message=update_succeed)
+    else:
+        return get_json_result(code=RetCode.SERVER_ERROR, message=f"Failed to forget message '{message_id}' in memory '{memory_id}'.")
+
+
+@manager.route("/<memory_id>:<message_id>", methods=["PUT"]) # noqa: F821
+@login_required
+@validate_request("status")
+async def update_message(memory_id: str, message_id: int):
+    req = await get_request_json()
+    status = req["status"]
+    if not isinstance(status, bool):
+        return get_error_argument_result("Status must be a boolean.")
+
+    memory = MemoryService.get_by_memory_id(memory_id)
+    if not memory:
+        return get_json_result(code=RetCode.NOT_FOUND, message=f"Memory '{memory_id}' not found.")
+
+    update_succeed = MessageService.update_message({"memory_id": memory_id, "message_id": int(message_id)}, {"status": status}, memory.tenant_id, memory_id)
+    if update_succeed:
+        return get_json_result(message=update_succeed)
+    else:
+        return get_json_result(code=RetCode.SERVER_ERROR, message=f"Failed to set status for message '{message_id}' in memory '{memory_id}'.")
+
+
+@manager.route("/search", methods=["GET"]) # noqa: F821
+@login_required
+async def search_message():
+    args = request.args
+    print(args, flush=True)
+    empty_fields = [f for f in ["memory_id", "query"] if not args.get(f)]
+    if empty_fields:
+        return get_error_argument_result(f"{', '.join(empty_fields)} can't be empty.")
+
+    memory_ids = args.getlist("memory_id")
+    query = args.get("query")
+    similarity_threshold = float(args.get("similarity_threshold", 0.2))
+    keywords_similarity_weight = float(args.get("keywords_similarity_weight", 0.7))
+    top_n = int(args.get("top_n", 5))
+    agent_id = args.get("agent_id", "")
+    session_id = args.get("session_id", "")
+
+    filter_dict = {
+        "memory_id": memory_ids,
+        "agent_id": agent_id,
+        "session_id": session_id
+    }
+    params = {
+        "query": query,
+        "similarity_threshold": similarity_threshold,
+        "keywords_similarity_weight": keywords_similarity_weight,
+        "top_n": top_n
+    }
+    res = memory_message_service.query_message(filter_dict, params)
+    return get_json_result(message=True, data=res)
+
+
+@manager.route("", methods=["GET"]) # noqa: F821
+@login_required
+async def get_messages():
+    args = request.args
+    memory_ids = args.getlist("memory_id")
+    agent_id = args.get("agent_id", "")
+    session_id = args.get("session_id", "")
+    limit = int(args.get("limit", 10))
+    if not memory_ids:
+        return get_error_argument_result("memory_ids is required.")
+    memory_list = MemoryService.get_by_ids(memory_ids)
+    uids = [memory.tenant_id for memory in memory_list]
+    res = MessageService.get_recent_messages(
+        uids,
+        memory_ids,
+        agent_id,
+        session_id,
+        limit
+    )
+    return get_json_result(message=True, data=res)
+
+
+@manager.route("/<memory_id>:<message_id>/content", methods=["GET"]) # noqa: F821
+@login_required
+async def get_message_content(memory_id:str, message_id: int):
+    memory = MemoryService.get_by_memory_id(memory_id)
+    if not memory:
+        return get_json_result(code=RetCode.NOT_FOUND, message=f"Memory '{memory_id}' not found.")
+
+    res = MessageService.get_by_message_id(memory_id, message_id, memory.tenant_id)
+    if res:
+        return get_json_result(message=True, data=res)
+    else:
+        return get_json_result(code=RetCode.NOT_FOUND, message=f"Message '{message_id}' in memory '{memory_id}' not found.")

api/apps/sdk/agents.py

@@ -14,20 +14,29 @@
 #  limitations under the License.
 #
 
+import asyncio
+import base64
+import hashlib
+import hmac
+import ipaddress
 import json
 import logging
 import time
 from typing import Any, cast
 
+import jwt
+
 from agent.canvas import Canvas
 from api.db import CanvasCategory
 from api.db.services.canvas_service import UserCanvasService
+from api.db.services.file_service import FileService
 from api.db.services.user_canvas_version import UserCanvasVersionService
 from common.constants import RetCode
 from common.misc_utils import get_uuid
 from api.utils.api_utils import get_data_error_result, get_error_data_result, get_json_result, get_request_json, token_required
 from api.utils.api_utils import get_result
 from quart import request, Response
+from rag.utils.redis_conn import REDIS_CONN
 
 
 @manager.route('/agents', methods=['GET'])  # noqa: F821
@@ -132,48 +141,785 @@ def delete_agent(tenant_id: str, agent_id: str):
     UserCanvasService.delete_by_id(agent_id)
     return get_json_result(data=True)
 
+@manager.route("/webhook/<agent_id>", methods=["POST", "GET", "PUT", "PATCH", "DELETE", "HEAD"])  # noqa: F821
+@manager.route("/webhook_test/<agent_id>",methods=["POST", "GET", "PUT", "PATCH", "DELETE", "HEAD"],)  # noqa: F821
+async def webhook(agent_id: str):
+    is_test = request.path.startswith("/api/v1/webhook_test")
+    start_ts = time.time()
 
-@manager.route('/webhook/<agent_id>', methods=['POST'])  # noqa: F821
-@token_required
-async def webhook(tenant_id: str, agent_id: str):
-    req = await get_request_json()
-    if not UserCanvasService.accessible(req["id"], tenant_id):
-        return get_json_result(
-            data=False, message='Only owner of canvas authorized for this operation.',
-            code=RetCode.OPERATING_ERROR)
-
-    e, cvs = UserCanvasService.get_by_id(req["id"])
-    if not e:
-        return get_data_error_result(message="canvas not found.")
-
-    if not isinstance(cvs.dsl, str):
-        cvs.dsl = json.dumps(cvs.dsl, ensure_ascii=False)
+    # 1. Fetch canvas by agent_id
+    exists, cvs = UserCanvasService.get_by_id(agent_id)
+    if not exists:
+        return get_data_error_result(code=RetCode.BAD_REQUEST,message="Canvas not found."),RetCode.BAD_REQUEST
 
+    # 2. Check canvas category
     if cvs.canvas_category == CanvasCategory.DataFlow:
-        return get_data_error_result(message="Dataflow can not be triggered by webhook.")
+        return get_data_error_result(code=RetCode.BAD_REQUEST,message="Dataflow can not be triggered by webhook."),RetCode.BAD_REQUEST
+
+    # 3. Load DSL from canvas
+    dsl = getattr(cvs, "dsl", None)
+    if not isinstance(dsl, dict):
+        return get_data_error_result(code=RetCode.BAD_REQUEST,message="Invalid DSL format."),RetCode.BAD_REQUEST
+
+    # 4. Check webhook configuration in DSL
+    components = dsl.get("components", {})
+    for k, _ in components.items():
+        cpn_obj = components[k]["obj"]
+        if cpn_obj["component_name"].lower() == "begin" and cpn_obj["params"]["mode"] == "Webhook":
+            webhook_cfg = cpn_obj["params"]
+
+    if not webhook_cfg:
+        return get_data_error_result(code=RetCode.BAD_REQUEST,message="Webhook not configured for this agent."),RetCode.BAD_REQUEST
+
+    # 5. Validate request method against webhook_cfg.methods
+    allowed_methods = webhook_cfg.get("methods", [])
+    request_method = request.method.upper()
+    if allowed_methods and request_method not in allowed_methods:
+        return get_data_error_result(
+            code=RetCode.BAD_REQUEST,message=f"HTTP method '{request_method}' not allowed for this webhook."
+        ),RetCode.BAD_REQUEST
+
+    # 6. Validate webhook security
+    async def validate_webhook_security(security_cfg: dict):
+        """Validate webhook security rules based on security configuration."""
+
+        if not security_cfg:
+            return  # No security config → allowed by default
+
+        # 1. Validate max body size
+        await _validate_max_body_size(security_cfg)
+
+        # 2. Validate IP whitelist
+        _validate_ip_whitelist(security_cfg)
+
+        # # 3. Validate rate limiting
+        _validate_rate_limit(security_cfg)
+
+        # 4. Validate authentication
+        auth_type = security_cfg.get("auth_type", "none")
+
+        if auth_type == "none":
+            return
+
+        if auth_type == "token":
+            _validate_token_auth(security_cfg)
+
+        elif auth_type == "basic":
+            _validate_basic_auth(security_cfg)
+
+        elif auth_type == "jwt":
+            _validate_jwt_auth(security_cfg)
+
+        else:
+            raise Exception(f"Unsupported auth_type: {auth_type}")
+
+    async def _validate_max_body_size(security_cfg):
+        """Check request size does not exceed max_body_size."""
+        max_size = security_cfg.get("max_body_size")
+        if not max_size:
+            return
+
+        # Convert "10MB" → bytes
+        units = {"kb": 1024, "mb": 1024**2}
+        size_str = max_size.lower()
+
+        for suffix, factor in units.items():
+            if size_str.endswith(suffix):
+                limit = int(size_str.replace(suffix, "")) * factor
+                break
+        else:
+            raise Exception("Invalid max_body_size format")
+        MAX_LIMIT = 10 * 1024 * 1024  # 10MB
+        if limit > MAX_LIMIT:
+            raise Exception("max_body_size exceeds maximum allowed size (10MB)")
+
+        content_length = request.content_length or 0
+        if content_length > limit:
+            raise Exception(f"Request body too large: {content_length} > {limit}")
+
+    def _validate_ip_whitelist(security_cfg):
+        """Allow only IPs listed in ip_whitelist."""
+        whitelist = security_cfg.get("ip_whitelist", [])
+        if not whitelist:
+            return
+
+        client_ip = request.remote_addr
+
+
+        for rule in whitelist:
+            if "/" in rule:
+                # CIDR notation
+                if ipaddress.ip_address(client_ip) in ipaddress.ip_network(rule, strict=False):
+                    return
+            else:
+                # Single IP
+                if client_ip == rule:
+                    return
+
+        raise Exception(f"IP {client_ip} is not allowed by whitelist")
+
+    def _validate_rate_limit(security_cfg):
+        """Simple in-memory rate limiting."""
+        rl = security_cfg.get("rate_limit")
+        if not rl:
+            return
+
+        limit = int(rl.get("limit", 60))
+        if limit <= 0:
+            raise Exception("rate_limit.limit must be > 0")
+        per = rl.get("per", "minute")
+
+        window = {
+            "second": 1,
+            "minute": 60,
+            "hour": 3600,
+            "day": 86400,
+        }.get(per)
+
+        if not window:
+            raise Exception(f"Invalid rate_limit.per: {per}")
+
+        capacity = limit
+        rate = limit / window
+        cost = 1
+
+        key = f"rl:tb:{agent_id}"
+        now = time.time()
+
+        try:
+            res = REDIS_CONN.lua_token_bucket(
+                keys=[key],
+                args=[capacity, rate, now, cost],
+                client=REDIS_CONN.REDIS,
+            )
+
+            allowed = int(res[0])
+            if allowed != 1:
+                raise Exception("Too many requests (rate limit exceeded)")
+
+        except Exception as e:
+            raise Exception(f"Rate limit error: {e}")
+
+    def _validate_token_auth(security_cfg):
+        """Validate header-based token authentication."""
+        token_cfg = security_cfg.get("token",{})
+        header = token_cfg.get("token_header")
+        token_value = token_cfg.get("token_value")
+
+        provided = request.headers.get(header)
+        if provided != token_value:
+            raise Exception("Invalid token authentication")
+
+    def _validate_basic_auth(security_cfg):
+        """Validate HTTP Basic Auth credentials."""
+        auth_cfg = security_cfg.get("basic_auth", {})
+        username = auth_cfg.get("username")
+        password = auth_cfg.get("password")
+
+        auth = request.authorization
+        if not auth or auth.username != username or auth.password != password:
+            raise Exception("Invalid Basic Auth credentials")
+
+    def _validate_jwt_auth(security_cfg):
+        """Validate JWT token in Authorization header."""
+        jwt_cfg = security_cfg.get("jwt", {})
+        secret = jwt_cfg.get("secret")
+        if not secret:
+            raise Exception("JWT secret not configured")
+
+        auth_header = request.headers.get("Authorization", "")
+        if not auth_header.startswith("Bearer "):
+            raise Exception("Missing Bearer token")
+
+        token = auth_header[len("Bearer "):].strip()
+        if not token:
+            raise Exception("Empty Bearer token")
+
+        alg = (jwt_cfg.get("algorithm") or "HS256").upper()
+
+        decode_kwargs = {
+            "key": secret,
+            "algorithms": [alg],
+        }
+        options = {}
+        if jwt_cfg.get("audience"):
+            decode_kwargs["audience"] = jwt_cfg["audience"]
+            options["verify_aud"] = True
+        else:
+            options["verify_aud"] = False
+
+        if jwt_cfg.get("issuer"):
+            decode_kwargs["issuer"] = jwt_cfg["issuer"]
+            options["verify_iss"] = True
+        else:
+            options["verify_iss"] = False
+        try:
+            decoded = jwt.decode(
+                token,
+                options=options,
+                **decode_kwargs,
+            )
+        except Exception as e:
+            raise Exception(f"Invalid JWT: {str(e)}")
+
+        raw_required_claims = jwt_cfg.get("required_claims", [])
+        if isinstance(raw_required_claims, str):
+            required_claims = [raw_required_claims]
+        elif isinstance(raw_required_claims, (list, tuple, set)):
+            required_claims = list(raw_required_claims)
+        else:
+            required_claims = []
+
+        required_claims = [
+            c for c in required_claims
+            if isinstance(c, str) and c.strip()
+        ]
+
+        RESERVED_CLAIMS = {"exp", "sub", "aud", "iss", "nbf", "iat"}
+        for claim in required_claims:
+            if claim in RESERVED_CLAIMS:
+                raise Exception(f"Reserved JWT claim cannot be required: {claim}")
+
+        for claim in required_claims:
+            if claim not in decoded:
+                raise Exception(f"Missing JWT claim: {claim}")
+
+        return decoded
 
     try:
-        canvas = Canvas(cvs.dsl, tenant_id, agent_id)
+        security_config=webhook_cfg.get("security", {})
+        await validate_webhook_security(security_config)
     except Exception as e:
-        return get_json_result(
-            data=False, message=str(e),
-            code=RetCode.EXCEPTION_ERROR)
+        return get_data_error_result(code=RetCode.BAD_REQUEST,message=str(e)),RetCode.BAD_REQUEST
+    if not isinstance(cvs.dsl, str):
+        dsl = json.dumps(cvs.dsl, ensure_ascii=False)
+    try:
+        canvas = Canvas(dsl, cvs.user_id, agent_id, canvas_id=agent_id)
+    except Exception as e:
+        resp=get_data_error_result(code=RetCode.BAD_REQUEST,message=str(e))
+        resp.status_code = RetCode.BAD_REQUEST
+        return resp
+
+    # 7. Parse request body
+    async def parse_webhook_request(content_type):
+        """Parse request based on content-type and return structured data."""
+
+        # 1. Query
+        query_data = {k: v for k, v in request.args.items()}
+
+        # 2. Headers
+        header_data = {k: v for k, v in request.headers.items()}
+
+        # 3. Body
+        ctype = request.headers.get("Content-Type", "").split(";")[0].strip()
+        if ctype and ctype != content_type:
+            raise ValueError(
+                f"Invalid Content-Type: expect '{content_type}', got '{ctype}'"
+            )
+
+        body_data: dict = {}
 
-    async def sse():
-        nonlocal canvas
         try:
-            async for ans in canvas.run(query=req.get("query", ""), files=req.get("files", []), user_id=req.get("user_id", tenant_id), webhook_payload=req):
-                yield "data:" + json.dumps(ans, ensure_ascii=False) + "\n\n"
+            if ctype == "application/json":
+                body_data = await request.get_json() or {}
 
-            cvs.dsl = json.loads(str(canvas))
-            UserCanvasService.update_by_id(req["id"], cvs.to_dict())
-        except Exception as e:
-            logging.exception(e)
-            yield "data:" + json.dumps({"code": 500, "message": str(e), "data": False}, ensure_ascii=False) + "\n\n"
+            elif ctype == "multipart/form-data":
+                nonlocal canvas
+                form = await request.form
+                files = await request.files
 
-    resp = Response(sse(), mimetype="text/event-stream")
-    resp.headers.add_header("Cache-control", "no-cache")
-    resp.headers.add_header("Connection", "keep-alive")
-    resp.headers.add_header("X-Accel-Buffering", "no")
-    resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
-    return resp
+                body_data = {}
+
+                for key, value in form.items():
+                    body_data[key] = value
+
+                if len(files) > 10:
+                    raise Exception("Too many uploaded files")
+                for key, file in files.items():
+                    desc = FileService.upload_info(
+                        cvs.user_id,           # user
+                        file,              # FileStorage
+                        None                   # url (None for webhook)
+                    )
+                    file_parsed= await canvas.get_files_async([desc])
+                    body_data[key] = file_parsed
+
+            elif ctype == "application/x-www-form-urlencoded":
+                form = await request.form
+                body_data = dict(form)
+
+            else:
+                # text/plain / octet-stream / empty / unknown
+                raw = await request.get_data()
+                if raw:
+                    try:
+                        body_data = json.loads(raw.decode("utf-8"))
+                    except Exception:
+                        body_data = {}
+                else:
+                    body_data = {}
+
+        except Exception:
+            body_data = {}
+
+        return {
+            "query": query_data,
+            "headers": header_data,
+            "body": body_data,
+            "content_type": ctype,
+        }
+
+    def extract_by_schema(data, schema, name="section"):
+        """
+        Extract only fields defined in schema.
+        Required fields must exist.
+        Optional fields default to type-based default values.
+        Type validation included.
+        """
+        props = schema.get("properties", {})
+        required = schema.get("required", [])
+
+        extracted = {}
+
+        for field, field_schema in props.items():
+            field_type = field_schema.get("type")
+
+            # 1. Required field missing
+            if field in required and field not in data:
+                raise Exception(f"{name} missing required field: {field}")
+
+            # 2. Optional → default value
+            if field not in data:
+                extracted[field] = default_for_type(field_type)
+                continue
+
+            raw_value = data[field]
+
+            # 3. Auto convert value
+            try:
+                value = auto_cast_value(raw_value, field_type)
+            except Exception as e:
+                raise Exception(f"{name}.{field} auto-cast failed: {str(e)}")
+
+            # 4. Type validation
+            if not validate_type(value, field_type):
+                raise Exception(
+                    f"{name}.{field} type mismatch: expected {field_type}, got {type(value).__name__}"
+                )
+
+            extracted[field] = value
+
+        return extracted
+
+
+    def default_for_type(t):
+        """Return default value for the given schema type."""
+        if t == "file":
+            return []
+        if t == "object":
+            return {}
+        if t == "boolean":
+            return False
+        if t == "number":
+            return 0
+        if t == "string":
+            return ""
+        if t and t.startswith("array"):
+            return []
+        if t == "null":
+            return None
+        return None
+
+    def auto_cast_value(value, expected_type):
+        """Convert string values into schema type when possible."""
+
+        # Non-string values already good
+        if not isinstance(value, str):
+            return value
+
+        v = value.strip()
+
+        # Boolean
+        if expected_type == "boolean":
+            if v.lower() in ["true", "1"]:
+                return True
+            if v.lower() in ["false", "0"]:
+                return False
+            raise Exception(f"Cannot convert '{value}' to boolean")
+
+        # Number
+        if expected_type == "number":
+            # integer
+            if v.isdigit() or (v.startswith("-") and v[1:].isdigit()):
+                return int(v)
+
+            # float
+            try:
+                return float(v)
+            except Exception:
+                raise Exception(f"Cannot convert '{value}' to number")
+
+        # Object
+        if expected_type == "object":
+            try:
+                parsed = json.loads(v)
+                if isinstance(parsed, dict):
+                    return parsed
+                else:
+                    raise Exception("JSON is not an object")
+            except Exception:
+                raise Exception(f"Cannot convert '{value}' to object")
+
+        # Array <T>
+        if expected_type.startswith("array"):
+            try:
+                parsed = json.loads(v)
+                if isinstance(parsed, list):
+                    return parsed
+                else:
+                    raise Exception("JSON is not an array")
+            except Exception:
+                raise Exception(f"Cannot convert '{value}' to array")
+
+        # String (accept original)
+        if expected_type == "string":
+            return value
+
+        # File
+        if expected_type == "file":
+            return value
+        # Default: do nothing
+        return value
+
+
+    def validate_type(value, t):
+        """Validate value type against schema type t."""
+        if t == "file":
+            return isinstance(value, list)
+
+        if t == "string":
+            return isinstance(value, str)
+
+        if t == "number":
+            return isinstance(value, (int, float))
+
+        if t == "boolean":
+            return isinstance(value, bool)
+
+        if t == "object":
+            return isinstance(value, dict)
+
+        # array<string> / array<number> / array<object>
+        if t.startswith("array"):
+            if not isinstance(value, list):
+                return False
+
+            if "<" in t and ">" in t:
+                inner = t[t.find("<") + 1 : t.find(">")]
+
+                # Check each element type
+                for item in value:
+                    if not validate_type(item, inner):
+                        return False
+
+            return True
+
+        return True
+    parsed = await parse_webhook_request(webhook_cfg.get("content_types"))
+    SCHEMA = webhook_cfg.get("schema", {"query": {}, "headers": {}, "body": {}})
+
+    # Extract strictly by schema
+    try:
+        query_clean  = extract_by_schema(parsed["query"],   SCHEMA.get("query", {}),  name="query")
+        header_clean = extract_by_schema(parsed["headers"], SCHEMA.get("headers", {}), name="headers")
+        body_clean   = extract_by_schema(parsed["body"],    SCHEMA.get("body", {}),    name="body")
+    except Exception as e:
+        return get_data_error_result(code=RetCode.BAD_REQUEST,message=str(e)),RetCode.BAD_REQUEST
+
+    clean_request = {
+        "query": query_clean,
+        "headers": header_clean,
+        "body": body_clean,
+        "input": parsed
+    }
+
+    execution_mode = webhook_cfg.get("execution_mode", "Immediately")
+    response_cfg = webhook_cfg.get("response", {})
+
+    def append_webhook_trace(agent_id: str, start_ts: float,event: dict, ttl=600):
+        key = f"webhook-trace-{agent_id}-logs"
+
+        raw = REDIS_CONN.get(key)
+        obj = json.loads(raw) if raw else {"webhooks": {}}
+
+        ws = obj["webhooks"].setdefault(
+            str(start_ts),
+            {"start_ts": start_ts, "events": []}
+        )
+
+        ws["events"].append({
+            "ts": time.time(),
+            **event
+        })
+
+        REDIS_CONN.set_obj(key, obj, ttl)
+
+    if execution_mode == "Immediately":
+        status = response_cfg.get("status", 200)
+        try:
+            status = int(status)
+        except (TypeError, ValueError):
+            return get_data_error_result(code=RetCode.BAD_REQUEST,message=str(f"Invalid response status code: {status}")),RetCode.BAD_REQUEST
+
+        if not (200 <= status <= 399):
+            return get_data_error_result(code=RetCode.BAD_REQUEST,message=str(f"Invalid response status code: {status}, must be between 200 and 399")),RetCode.BAD_REQUEST
+
+        body_tpl = response_cfg.get("body_template", "")
+
+        def parse_body(body: str):
+            if not body:
+                return None, "application/json"
+
+            try:
+                parsed = json.loads(body)
+                return parsed, "application/json"
+            except (json.JSONDecodeError, TypeError):
+                return body, "text/plain"
+
+
+        body, content_type = parse_body(body_tpl)
+        resp = Response(
+            json.dumps(body, ensure_ascii=False) if content_type == "application/json" else body,
+            status=status,
+            content_type=content_type,
+        )
+
+        async def background_run():
+            try:
+                async for ans in canvas.run(
+                    query="",
+                    user_id=cvs.user_id,
+                    webhook_payload=clean_request
+                ):
+                    if is_test:
+                        append_webhook_trace(agent_id, start_ts, ans)
+
+                if is_test:
+                    append_webhook_trace(
+                        agent_id,
+                        start_ts,
+                        {
+                            "event": "finished",
+                            "elapsed_time": time.time() - start_ts,
+                            "success": True,
+                        }
+                    )
+
+                cvs.dsl = json.loads(str(canvas))
+                UserCanvasService.update_by_id(cvs.user_id, cvs.to_dict())
+
+            except Exception as e:
+                logging.exception("Webhook background run failed")
+                if is_test:
+                    try:
+                        append_webhook_trace(
+                            agent_id,
+                            start_ts,
+                            {
+                                "event": "error",
+                                "message": str(e),
+                                "error_type": type(e).__name__,
+                            }
+                        )
+                        append_webhook_trace(
+                            agent_id,
+                            start_ts,
+                            {
+                                "event": "finished",
+                                "elapsed_time": time.time() - start_ts,
+                                "success": False,
+                            }
+                        )
+                    except Exception:
+                        logging.exception("Failed to append webhook trace")
+
+        asyncio.create_task(background_run())
+        return resp
+    else:
+        async def sse():
+            nonlocal canvas
+            contents: list[str] = []
+            status = 200
+            try:
+                async for ans in canvas.run(
+                    query="",
+                    user_id=cvs.user_id,
+                    webhook_payload=clean_request,
+                ):
+                    if ans["event"] == "message":
+                        content = ans["data"]["content"]
+                        if ans["data"].get("start_to_think", False):
+                            content = "<think>"
+                        elif ans["data"].get("end_to_think", False):
+                            content = "</think>"
+                        if content:
+                            contents.append(content)
+                    if ans["event"] == "message_end":
+                        status = int(ans["data"].get("status", status))
+                    if is_test:
+                        append_webhook_trace(
+                            agent_id,
+                            start_ts,
+                            ans
+                        )
+                if is_test:
+                    append_webhook_trace(
+                        agent_id,
+                        start_ts,
+                        {
+                            "event": "finished",
+                            "elapsed_time": time.time() - start_ts,
+                            "success": True,
+                        }
+                    )
+                final_content = "".join(contents)
+                return {
+                    "message": final_content,
+                    "success": True,
+                    "code":  status,
+                }
+
+            except Exception as e:
+                if is_test:
+                    append_webhook_trace(
+                        agent_id,
+                        start_ts,
+                        {
+                            "event": "error",
+                            "message": str(e),
+                            "error_type": type(e).__name__,
+                        }
+                    )
+                    append_webhook_trace(
+                        agent_id,
+                        start_ts,
+                        {
+                            "event": "finished",
+                            "elapsed_time": time.time() - start_ts,
+                            "success": False,
+                        }
+                    )
+                return {"code": 400, "message": str(e),"success":False}
+
+        result = await sse()
+        return Response(
+            json.dumps(result),
+            status=result["code"],
+            mimetype="application/json",
+        )
+
+
+@manager.route("/webhook_trace/<agent_id>", methods=["GET"])  # noqa: F821
+async def webhook_trace(agent_id: str):
+    def encode_webhook_id(start_ts: str) -> str:
+        WEBHOOK_ID_SECRET = "webhook_id_secret"
+        sig = hmac.new(
+            WEBHOOK_ID_SECRET.encode("utf-8"),
+            start_ts.encode("utf-8"),
+            hashlib.sha256,
+        ).digest()
+        return base64.urlsafe_b64encode(sig).decode("utf-8").rstrip("=")
+
+    def decode_webhook_id(enc_id: str, webhooks: dict) -> str | None:
+        for ts in webhooks.keys():
+            if encode_webhook_id(ts) == enc_id:
+                return ts
+        return None
+    since_ts = request.args.get("since_ts", type=float)
+    webhook_id = request.args.get("webhook_id")
+
+    key = f"webhook-trace-{agent_id}-logs"
+    raw = REDIS_CONN.get(key)
+
+    if since_ts is None:
+        now = time.time()
+        return get_json_result(
+            data={
+                "webhook_id": None,
+                "events": [],
+                "next_since_ts": now,
+                "finished": False,
+            }
+        )
+
+    if not raw:
+        return get_json_result(
+            data={
+                "webhook_id": None,
+                "events": [],
+                "next_since_ts": since_ts,
+                "finished": False,
+            }
+        )
+
+    obj = json.loads(raw)
+    webhooks = obj.get("webhooks", {})
+
+    if webhook_id is None:
+        candidates = [
+            float(k) for k in webhooks.keys() if float(k) > since_ts
+        ]
+
+        if not candidates:
+            return get_json_result(
+                data={
+                    "webhook_id": None,
+                    "events": [],
+                    "next_since_ts": since_ts,
+                    "finished": False,
+                }
+            )
+
+        start_ts = min(candidates)
+        real_id = str(start_ts)
+        webhook_id = encode_webhook_id(real_id)
+
+        return get_json_result(
+            data={
+                "webhook_id": webhook_id,
+                "events": [],
+                "next_since_ts": start_ts,
+                "finished": False,
+            }
+        )
+
+    real_id = decode_webhook_id(webhook_id, webhooks)
+
+    if not real_id:
+        return get_json_result(
+            data={
+                "webhook_id": webhook_id,
+                "events": [],
+                "next_since_ts": since_ts,
+                "finished": True,
+            }
+        )
+
+    ws = webhooks.get(str(real_id))
+    events = ws.get("events", [])
+    new_events = [e for e in events if e.get("ts", 0) > since_ts]
+
+    next_ts = since_ts
+    for e in new_events:
+        next_ts = max(next_ts, e["ts"])
+
+    finished = any(e.get("event") == "finished" for e in new_events)
+
+    return get_json_result(
+        data={
+            "webhook_id": webhook_id,
+            "events": new_events,
+            "next_since_ts": next_ts,
+            "finished": finished,
+        }
+    )

api/apps/sdk/chat.py

@@ -92,7 +92,7 @@ async def create(tenant_id):
     req["tenant_id"] = tenant_id
     # prompt more parameter
     default_prompt = {
-        "system": """You are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence "The answer you are looking for is not found in the knowledge base!" Answers need to consider chat history.
+        "system": """You are an intelligent assistant. Please summarize the content of the dataset to answer the question. Please list the data in the dataset and answer in detail. When all dataset content is irrelevant to the question, your answer must include the sentence "The answer you are looking for is not found in the dataset!" Answers need to consider chat history.
       Here is the knowledge base:
       {knowledge}
       The above is the knowledge base.""",
@@ -174,7 +174,9 @@ async def update(tenant_id, chat_id):
             req["llm_id"] = llm.pop("model_name")
             if req.get("llm_id") is not None:
                 llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(req["llm_id"])
-                if not TenantLLMService.query(tenant_id=tenant_id, llm_name=llm_name, llm_factory=llm_factory, model_type="chat"):
+                model_type = llm.pop("model_type")
+                model_type = model_type if model_type in ["chat", "image2text"] else "chat"
+                if not TenantLLMService.query(tenant_id=tenant_id, llm_name=llm_name, llm_factory=llm_factory, model_type=model_type):
                     return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
         req["llm_setting"] = req.pop("llm")
     e, tenant = TenantService.get_by_id(tenant_id)
@@ -250,7 +252,6 @@ async def delete_chats(tenant_id):
             continue
         temp_dict = {"status": StatusEnum.INVALID.value}
         success_count += DialogService.update_by_id(id, temp_dict)
-        print(success_count, "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$", flush=True)
 
     if errors:
         if success_count > 0:
@@ -286,7 +287,7 @@ def list_chat(tenant_id):
     chats = DialogService.get_list(tenant_id, page_number, items_per_page, orderby, desc, id, name)
     if not chats:
         return get_result(data=[])
-    list_assts = []
+    list_assistants = []
     key_mapping = {
         "parameters": "variables",
         "prologue": "opener",
@@ -320,5 +321,5 @@ def list_chat(tenant_id):
         del res["kb_ids"]
         res["datasets"] = kb_list
         res["avatar"] = res.pop("icon")
-        list_assts.append(res)
-    return get_result(data=list_assts)
+        list_assistants.append(res)
+    return get_result(data=list_assistants)

api/apps/sdk/dataset.py

@@ -495,7 +495,7 @@ def knowledge_graph(tenant_id, dataset_id):
     }
 
     obj = {"graph": {}, "mind_map": {}}
-    if not settings.docStoreConn.indexExist(search.index_name(kb.tenant_id), dataset_id):
+    if not settings.docStoreConn.index_exist(search.index_name(kb.tenant_id), dataset_id):
         return get_result(data=obj)
     sres = settings.retriever.search(req, search.index_name(kb.tenant_id), [dataset_id])
     if not len(sres.ids):

api/apps/sdk/dify_retrieval.py

@@ -20,9 +20,9 @@ from quart import jsonify
 from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
+from common.metadata_utils import meta_filter, convert_conditions
 from api.utils.api_utils import apikey_required, build_error_result, get_request_json, validate_request
 from rag.app.tag import label_question
-from api.db.services.dialog_service import meta_filter, convert_conditions
 from common.constants import RetCode, LLMType
 from common import settings
 

api/apps/sdk/doc.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 #
 import datetime
+import json
 import logging
 import pathlib
 import re
@@ -34,7 +35,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.task_service import TaskService, queue_tasks, cancel_all_task_of
-from api.db.services.dialog_service import meta_filter, convert_conditions
+from common.metadata_utils import meta_filter, convert_conditions
 from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_parser_config, get_result, server_error_response, token_required, \
     get_request_json
 from rag.app.qa import beAdoc, rmPrefix
@@ -236,7 +237,7 @@ async def update_doc(tenant_id, dataset_id, document_id):
         return get_error_data_result(message="You don't own the dataset.")
     e, kb = KnowledgebaseService.get_by_id(dataset_id)
     if not e:
-        return get_error_data_result(message="Can't find this knowledgebase!")
+        return get_error_data_result(message="Can't find this dataset!")
     doc = DocumentService.query(kb_id=dataset_id, id=document_id)
     if not doc:
         return get_error_data_result(message="The dataset doesn't own the document.")
@@ -551,13 +552,29 @@ def list_docs(dataset_id, tenant_id):
     run_status           = q.getlist("run")
     create_time_from     = int(q.get("create_time_from", 0))
     create_time_to       = int(q.get("create_time_to", 0))
+    metadata_condition_raw = q.get("metadata_condition")
+    metadata_condition = {}
+    if metadata_condition_raw:
+        try:
+            metadata_condition = json.loads(metadata_condition_raw)
+        except Exception:
+            return get_error_data_result(message="metadata_condition must be valid JSON.")
+    if metadata_condition and not isinstance(metadata_condition, dict):
+        return get_error_data_result(message="metadata_condition must be an object.")
 
-    # map run status (accept text or numeric) - align with API parameter
+    # map run status (text or numeric) - align with API parameter
     run_status_text_to_numeric = {"UNSTART": "0", "RUNNING": "1", "CANCEL": "2", "DONE": "3", "FAIL": "4"}
     run_status_converted = [run_status_text_to_numeric.get(v, v) for v in run_status]
 
+    doc_ids_filter = None
+    if metadata_condition:
+        metas = DocumentService.get_flatted_meta_by_kbs([dataset_id])
+        doc_ids_filter = meta_filter(metas, convert_conditions(metadata_condition), metadata_condition.get("logic", "and"))
+        if metadata_condition.get("conditions") and not doc_ids_filter:
+            return get_result(data={"total": 0, "docs": []})
+
     docs, total = DocumentService.get_list(
-        dataset_id, page, page_size, orderby, desc, keywords, document_id, name, suffix, run_status_converted
+        dataset_id, page, page_size, orderby, desc, keywords, document_id, name, suffix, run_status_converted, doc_ids_filter
     )
 
     # time range filter (0 means no bound)
@@ -586,6 +603,70 @@ def list_docs(dataset_id, tenant_id):
 
     return get_result(data={"total": total, "docs": output_docs})
 
+
+@manager.route("/datasets/<dataset_id>/metadata/summary", methods=["GET"])  # noqa: F821
+@token_required
+def metadata_summary(dataset_id, tenant_id):
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}. ")
+
+    try:
+        summary = DocumentService.get_metadata_summary(dataset_id)
+        return get_result(data={"summary": summary})
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/datasets/<dataset_id>/metadata/update", methods=["POST"])  # noqa: F821
+@token_required
+async def metadata_batch_update(dataset_id, tenant_id):
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}. ")
+
+    req = await get_request_json()
+    selector = req.get("selector", {}) or {}
+    updates = req.get("updates", []) or []
+    deletes = req.get("deletes", []) or []
+
+    if not isinstance(selector, dict):
+        return get_error_data_result(message="selector must be an object.")
+    if not isinstance(updates, list) or not isinstance(deletes, list):
+        return get_error_data_result(message="updates and deletes must be lists.")
+
+    metadata_condition = selector.get("metadata_condition", {}) or {}
+    if metadata_condition and not isinstance(metadata_condition, dict):
+        return get_error_data_result(message="metadata_condition must be an object.")
+
+    document_ids = selector.get("document_ids", []) or []
+    if document_ids and not isinstance(document_ids, list):
+        return get_error_data_result(message="document_ids must be a list.")
+
+    for upd in updates:
+        if not isinstance(upd, dict) or not upd.get("key") or "value" not in upd:
+            return get_error_data_result(message="Each update requires key and value.")
+    for d in deletes:
+        if not isinstance(d, dict) or not d.get("key"):
+            return get_error_data_result(message="Each delete requires key.")
+
+    kb_doc_ids = KnowledgebaseService.list_documents_by_ids([dataset_id])
+    target_doc_ids = set(kb_doc_ids)
+    if document_ids:
+        invalid_ids = set(document_ids) - set(kb_doc_ids)
+        if invalid_ids:
+            return get_error_data_result(message=f"These documents do not belong to dataset {dataset_id}: {', '.join(invalid_ids)}")
+        target_doc_ids = set(document_ids)
+
+    if metadata_condition:
+        metas = DocumentService.get_flatted_meta_by_kbs([dataset_id])
+        filtered_ids = set(meta_filter(metas, convert_conditions(metadata_condition), metadata_condition.get("logic", "and")))
+        target_doc_ids = target_doc_ids & filtered_ids
+        if metadata_condition.get("conditions") and not target_doc_ids:
+            return get_result(data={"updated": 0, "matched_docs": 0})
+
+    target_doc_ids = list(target_doc_ids)
+    updated = DocumentService.batch_update_metadata(dataset_id, target_doc_ids, updates, deletes)
+    return get_result(data={"updated": updated, "matched_docs": len(target_doc_ids)})
+
 @manager.route("/datasets/<dataset_id>/documents", methods=["DELETE"])  # noqa: F821
 @token_required
 async def delete(tenant_id, dataset_id):
@@ -890,7 +971,7 @@ def list_chunks(tenant_id, dataset_id, document_id):
         type: string
         required: false
         default: ""
-        description: Chunk Id.
+        description: Chunk id.
       - in: header
         name: Authorization
         type: string
@@ -999,7 +1080,7 @@ def list_chunks(tenant_id, dataset_id, document_id):
         res["chunks"].append(final_chunk)
         _ = Chunk(**final_chunk)
 
-    elif settings.docStoreConn.indexExist(search.index_name(tenant_id), dataset_id):
+    elif settings.docStoreConn.index_exist(search.index_name(tenant_id), dataset_id):
         sres = settings.retriever.search(query, search.index_name(tenant_id), [dataset_id], emb_mdl=None, highlight=True)
         res["total"] = sres.total
         for id in sres.ids:
@@ -1468,11 +1549,11 @@ async def retrieval_test(tenant_id):
             rerank_mdl = LLMBundle(kb.tenant_id, LLMType.RERANK, llm_name=req["rerank_id"])
 
         if langs:
-            question = cross_languages(kb.tenant_id, None, question, langs)
+            question = await cross_languages(kb.tenant_id, None, question, langs)
 
         if req.get("keyword", False):
             chat_mdl = LLMBundle(kb.tenant_id, LLMType.CHAT)
-            question += keyword_extraction(chat_mdl, question)
+            question += await keyword_extraction(chat_mdl, question)
 
         ranks = settings.retriever.retrieval(
             question,

api/apps/sdk/files.py

@@ -14,7 +14,7 @@
 #  limitations under the License.
 #
 
-
+import asyncio
 import pathlib
 import re
 from quart import request, make_response
@@ -29,9 +29,11 @@ from api.db import FileType
 from api.db.services import duplicate_name
 from api.db.services.file_service import FileService
 from api.utils.file_utils import filename_type
+from api.utils.web_utils import CONTENT_TYPE_MAP
 from common import settings
 from common.constants import RetCode
 
+
 @manager.route('/file/upload', methods=['POST'])  # noqa: F821
 @token_required
 async def upload(tenant_id):
@@ -203,7 +205,8 @@ async def create(tenant_id):
         if not FileService.is_parent_folder_exist(pf_id):
             return get_json_result(data=False, message="Parent Folder Doesn't Exist!", code=RetCode.BAD_REQUEST)
         if FileService.query(name=req["name"], parent_id=pf_id):
-            return get_json_result(data=False, message="Duplicated folder name in the same folder.", code=409)
+            return get_json_result(data=False, message="Duplicated folder name in the same folder.",
+                                   code=RetCode.CONFLICT)
 
         if input_file_type == FileType.FOLDER.value:
             file_type = FileType.FOLDER.value
@@ -563,11 +566,13 @@ async def rename(tenant_id):
 
         if file.type != FileType.FOLDER.value and pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
                 file.name.lower()).suffix:
-            return get_json_result(data=False, message="The extension of file can't be changed", code=RetCode.BAD_REQUEST)
+            return get_json_result(data=False, message="The extension of file can't be changed",
+                                   code=RetCode.BAD_REQUEST)
 
         for existing_file in FileService.query(name=req["name"], pf_id=file.parent_id):
             if existing_file.name == req["name"]:
-                return get_json_result(data=False, message="Duplicated file name in the same folder.", code=409)
+                return get_json_result(data=False, message="Duplicated file name in the same folder.",
+                                       code=RetCode.CONFLICT)
 
         if not FileService.update_by_id(req["file_id"], {"name": req["name"]}):
             return get_json_result(message="Database error (File rename)!", code=RetCode.SERVER_ERROR)
@@ -630,6 +635,21 @@ async def get(tenant_id, file_id):
         return server_error_response(e)
 
 
+@manager.route("/file/download/<attachment_id>", methods=["GET"])  # noqa: F821
+@token_required
+async def download_attachment(tenant_id, attachment_id):
+    try:
+        ext = request.args.get("ext", "markdown")
+        data = await asyncio.to_thread(settings.STORAGE_IMPL.get, tenant_id, attachment_id)
+        response = await make_response(data)
+        response.headers.set("Content-Type", CONTENT_TYPE_MAP.get(ext, f"application/{ext}"))
+
+        return response
+
+    except Exception as e:
+        return server_error_response(e)
+
+
 @manager.route('/file/mv', methods=['POST'])  # noqa: F821
 @token_required
 async def move(tenant_id):
@@ -729,7 +749,7 @@ async def convert(tenant_id):
                     e, kb = KnowledgebaseService.get_by_id(kb_id)
                     if not e:
                         return get_json_result(
-                            message="Can't find this knowledgebase!", code=RetCode.NOT_FOUND)
+                            message="Can't find this dataset!", code=RetCode.NOT_FOUND)
                     e, file = FileService.get_by_id(id)
                     if not e:
                         return get_json_result(

api/apps/sdk/session.py

@@ -13,8 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import asyncio
 import json
+import copy
 import re
 import time
 
@@ -26,12 +26,14 @@ from api.db.db_models import APIToken
 from api.db.services.api_service import API4ConversationService
 from api.db.services.canvas_service import UserCanvasService, completion_openai
 from api.db.services.canvas_service import completion as agent_completion
-from api.db.services.conversation_service import ConversationService, iframe_completion
-from api.db.services.conversation_service import completion as rag_completion
-from api.db.services.dialog_service import DialogService, ask, chat, gen_mindmap, meta_filter
+from api.db.services.conversation_service import ConversationService
+from api.db.services.conversation_service import async_iframe_completion as iframe_completion
+from api.db.services.conversation_service import async_completion as rag_completion
+from api.db.services.dialog_service import DialogService, async_ask, async_chat, gen_mindmap
 from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
+from common.metadata_utils import apply_meta_data_filter, convert_conditions, meta_filter
 from api.db.services.search_service import SearchService
 from api.db.services.user_service import UserTenantService
 from common.misc_utils import get_uuid
@@ -39,10 +41,11 @@ from api.utils.api_utils import check_duplicate_ids, get_data_openai, get_error_
     get_result, get_request_json, server_error_response, token_required, validate_request
 from rag.app.tag import label_question
 from rag.prompts.template import load_prompt
-from rag.prompts.generator import cross_languages, gen_meta_filter, keyword_extraction, chunks_format
+from rag.prompts.generator import cross_languages, keyword_extraction, chunks_format
 from common.constants import RetCode, LLMType, StatusEnum
 from common import settings
 
+
 @manager.route("/chats/<chat_id>/sessions", methods=["POST"])  # noqa: F821
 @token_required
 async def create(tenant_id, chat_id):
@@ -85,7 +88,7 @@ async def create_agent_session(tenant_id, agent_id):
         cvs.dsl = json.dumps(cvs.dsl, ensure_ascii=False)
 
     session_id = get_uuid()
-    canvas = Canvas(cvs.dsl, tenant_id, agent_id)
+    canvas = Canvas(cvs.dsl, tenant_id, agent_id, canvas_id=cvs.id)
     canvas.reset()
 
     cvs.dsl = json.loads(str(canvas))
@@ -126,11 +129,33 @@ async def chat_completion(tenant_id, chat_id):
         req = {"question": ""}
     if not req.get("session_id"):
         req["question"] = ""
-    if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
+    dia = DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value)
+    if not dia:
         return get_error_data_result(f"You don't own the chat {chat_id}")
+    dia = dia[0]
     if req.get("session_id"):
         if not ConversationService.query(id=req["session_id"], dialog_id=chat_id):
             return get_error_data_result(f"You don't own the session {req['session_id']}")
+
+    metadata_condition = req.get("metadata_condition") or {}
+    if metadata_condition and not isinstance(metadata_condition, dict):
+        return get_error_data_result(message="metadata_condition must be an object.")
+
+    if metadata_condition and req.get("question"):
+        metas = DocumentService.get_meta_by_kbs(dia.kb_ids or [])
+        filtered_doc_ids = meta_filter(
+            metas,
+            convert_conditions(metadata_condition),
+            metadata_condition.get("logic", "and"),
+        )
+        if metadata_condition.get("conditions") and not filtered_doc_ids:
+            filtered_doc_ids = ["-999"]
+
+        if filtered_doc_ids:
+            req["doc_ids"] = ",".join(filtered_doc_ids)
+        else:
+            req.pop("doc_ids", None)
+
     if req.get("stream", True):
         resp = Response(rag_completion(tenant_id, chat_id, **req), mimetype="text/event-stream")
         resp.headers.add_header("Cache-control", "no-cache")
@@ -141,7 +166,7 @@ async def chat_completion(tenant_id, chat_id):
         return resp
     else:
         answer = None
-        for ans in rag_completion(tenant_id, chat_id, **req):
+        async for ans in rag_completion(tenant_id, chat_id, **req):
             answer = ans
             break
         return get_result(data=answer)
@@ -193,7 +218,19 @@ async def chat_completion_openai_like(tenant_id, chat_id):
             {"role": "user", "content": "Can you tell me how to install neovim"},
         ],
         stream=stream,
-        extra_body={"reference": reference}
+        extra_body={
+            "reference": reference,
+            "metadata_condition": {
+                "logic": "and",
+                "conditions": [
+                    {
+                        "name": "author",
+                        "comparison_operator": "is",
+                        "value": "bob"
+                    }
+                ]
+            }
+        }
     )
 
     if stream:
@@ -209,7 +246,11 @@ async def chat_completion_openai_like(tenant_id, chat_id):
     """
     req = await get_request_json()
 
-    need_reference = bool(req.get("reference", False))
+    extra_body = req.get("extra_body") or {}
+    if extra_body and not isinstance(extra_body, dict):
+        return get_error_data_result("extra_body must be an object.")
+
+    need_reference = bool(extra_body.get("reference", False))
 
     messages = req.get("messages", [])
     # To prevent empty [] input
@@ -227,6 +268,22 @@ async def chat_completion_openai_like(tenant_id, chat_id):
         return get_error_data_result(f"You don't own the chat {chat_id}")
     dia = dia[0]
 
+    metadata_condition = extra_body.get("metadata_condition") or {}
+    if metadata_condition and not isinstance(metadata_condition, dict):
+        return get_error_data_result(message="metadata_condition must be an object.")
+
+    doc_ids_str = None
+    if metadata_condition:
+        metas = DocumentService.get_meta_by_kbs(dia.kb_ids or [])
+        filtered_doc_ids = meta_filter(
+            metas,
+            convert_conditions(metadata_condition),
+            metadata_condition.get("logic", "and"),
+        )
+        if metadata_condition.get("conditions") and not filtered_doc_ids:
+            filtered_doc_ids = ["-999"]
+        doc_ids_str = ",".join(filtered_doc_ids) if filtered_doc_ids else None
+
     # Filter system and non-sense assistant messages
     msg = []
     for m in messages:
@@ -245,7 +302,7 @@ async def chat_completion_openai_like(tenant_id, chat_id):
         # The value for the usage field on all chunks except for the last one will be null.
         # The usage field on the last chunk contains token usage statistics for the entire request.
         # The choices field on the last chunk will always be an empty array [].
-        def streamed_response_generator(chat_id, dia, msg):
+        async def streamed_response_generator(chat_id, dia, msg):
             token_used = 0
             answer_cache = ""
             reasoning_cache = ""
@@ -274,14 +331,17 @@ async def chat_completion_openai_like(tenant_id, chat_id):
             }
 
             try:
-                for ans in chat(dia, msg, True, toolcall_session=toolcall_session, tools=tools, quote=need_reference):
+                chat_kwargs = {"toolcall_session": toolcall_session, "tools": tools, "quote": need_reference}
+                if doc_ids_str:
+                    chat_kwargs["doc_ids"] = doc_ids_str
+                async for ans in async_chat(dia, msg, True, **chat_kwargs):
                     last_ans = ans
                     answer = ans["answer"]
 
                     reasoning_match = re.search(r"<think>(.*?)</think>", answer, flags=re.DOTALL)
                     if reasoning_match:
                         reasoning_part = reasoning_match.group(1)
-                        content_part = answer[reasoning_match.end():]
+                        content_part = answer[reasoning_match.end() :]
                     else:
                         reasoning_part = ""
                         content_part = answer
@@ -326,8 +386,7 @@ async def chat_completion_openai_like(tenant_id, chat_id):
             response["choices"][0]["delta"]["content"] = None
             response["choices"][0]["delta"]["reasoning_content"] = None
             response["choices"][0]["finish_reason"] = "stop"
-            response["usage"] = {"prompt_tokens": len(prompt), "completion_tokens": token_used,
-                                 "total_tokens": len(prompt) + token_used}
+            response["usage"] = {"prompt_tokens": len(prompt), "completion_tokens": token_used, "total_tokens": len(prompt) + token_used}
             if need_reference:
                 response["choices"][0]["delta"]["reference"] = chunks_format(last_ans.get("reference", []))
                 response["choices"][0]["delta"]["final_content"] = last_ans.get("answer", "")
@@ -342,7 +401,10 @@ async def chat_completion_openai_like(tenant_id, chat_id):
         return resp
     else:
         answer = None
-        for ans in chat(dia, msg, False, toolcall_session=toolcall_session, tools=tools, quote=need_reference):
+        chat_kwargs = {"toolcall_session": toolcall_session, "tools": tools, "quote": need_reference}
+        if doc_ids_str:
+            chat_kwargs["doc_ids"] = doc_ids_str
+        async for ans in async_chat(dia, msg, False, **chat_kwargs):
             # focus answer content only
             answer = ans
             break
@@ -386,7 +448,7 @@ async def chat_completion_openai_like(tenant_id, chat_id):
 @token_required
 async def agents_completion_openai_compatibility(tenant_id, agent_id):
     req = await get_request_json()
-    tiktokenenc = tiktoken.get_encoding("cl100k_base")
+    tiktoken_encode = tiktoken.get_encoding("cl100k_base")
     messages = req.get("messages", [])
     if not messages:
         return get_error_data_result("You must provide at least one message.")
@@ -394,7 +456,7 @@ async def agents_completion_openai_compatibility(tenant_id, agent_id):
         return get_error_data_result(f"You don't own the agent {agent_id}")
 
     filtered_messages = [m for m in messages if m["role"] in ["user", "assistant"]]
-    prompt_tokens = sum(len(tiktokenenc.encode(m["content"])) for m in filtered_messages)
+    prompt_tokens = sum(len(tiktoken_encode.encode(m["content"])) for m in filtered_messages)
     if not filtered_messages:
         return jsonify(
             get_data_openai(
@@ -402,7 +464,7 @@ async def agents_completion_openai_compatibility(tenant_id, agent_id):
                 content="No valid messages found (user or assistant).",
                 finish_reason="stop",
                 model=req.get("model", ""),
-                completion_tokens=len(tiktokenenc.encode("No valid messages found (user or assistant).")),
+                completion_tokens=len(tiktoken_encode.encode("No valid messages found (user or assistant).")),
                 prompt_tokens=prompt_tokens,
             )
         )
@@ -439,15 +501,19 @@ async def agents_completion_openai_compatibility(tenant_id, agent_id):
             ):
             return jsonify(response)
 
+        return None
+
 
 @manager.route("/agents/<agent_id>/completions", methods=["POST"])  # noqa: F821
 @token_required
 async def agent_completions(tenant_id, agent_id):
     req = await get_request_json()
+    return_trace = bool(req.get("return_trace", False))
 
     if req.get("stream", True):
 
         async def generate():
+            trace_items = []
             async for answer in agent_completion(tenant_id=tenant_id, agent_id=agent_id, **req):
                 if isinstance(answer, str):
                     try:
@@ -455,7 +521,21 @@ async def agent_completions(tenant_id, agent_id):
                     except Exception:
                         continue
 
-                if ans.get("event") not in ["message", "message_end"]:
+                event = ans.get("event")
+                if event == "node_finished":
+                    if return_trace:
+                        data = ans.get("data", {})
+                        trace_items.append(
+                            {
+                                "component_id": data.get("component_id"),
+                                "trace": [copy.deepcopy(data)],
+                            }
+                        )
+                        ans.setdefault("data", {})["trace"] = trace_items
+                        answer = "data:" + json.dumps(ans, ensure_ascii=False) + "\n\n"
+                    yield answer
+
+                if event not in ["message", "message_end"]:
                     continue
 
                 yield answer
@@ -472,6 +552,7 @@ async def agent_completions(tenant_id, agent_id):
     full_content = ""
     reference = {}
     final_ans = ""
+    trace_items = []
     async for answer in agent_completion(tenant_id=tenant_id, agent_id=agent_id, **req):
         try:
             ans = json.loads(answer[5:])
@@ -482,11 +563,22 @@ async def agent_completions(tenant_id, agent_id):
             if ans.get("data", {}).get("reference", None):
                 reference.update(ans["data"]["reference"])
 
+            if return_trace and ans.get("event") == "node_finished":
+                data = ans.get("data", {})
+                trace_items.append(
+                    {
+                        "component_id": data.get("component_id"),
+                        "trace": [copy.deepcopy(data)],
+                    }
+                )
+
             final_ans = ans
         except Exception as e:
             return get_result(data=f"**ERROR**: {str(e)}")
     final_ans["data"]["content"] = full_content
     final_ans["data"]["reference"] = reference
+    if return_trace and final_ans:
+        final_ans["data"]["trace"] = trace_items
     return get_result(data=final_ans)
 
 
@@ -733,10 +825,10 @@ async def ask_about(tenant_id):
             return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
     uid = tenant_id
 
-    def stream():
+    async def stream():
         nonlocal req, uid
         try:
-            for ans in ask(req["question"], req["kb_ids"], uid):
+            async for ans in async_ask(req["question"], req["kb_ids"], uid):
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
         except Exception as e:
             yield "data:" + json.dumps(
@@ -827,9 +919,10 @@ async def chatbot_completions(dialog_id):
         resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
         return resp
 
-    for answer in iframe_completion(dialog_id, **req):
+    async for answer in iframe_completion(dialog_id, **req):
         return get_result(data=answer)
 
+    return None
 
 @manager.route("/chatbots/<dialog_id>/info", methods=["GET"])  # noqa: F821
 async def chatbots_inputs(dialog_id):
@@ -877,6 +970,7 @@ async def agent_bot_completions(agent_id):
     async for answer in agent_completion(objs[0].tenant_id, agent_id, **req):
         return get_result(data=answer)
 
+    return None
 
 @manager.route("/agentbots/<agent_id>/inputs", methods=["GET"])  # noqa: F821
 async def begin_inputs(agent_id):
@@ -892,7 +986,7 @@ async def begin_inputs(agent_id):
     if not e:
         return get_error_data_result(f"Can't find agent by ID: {agent_id}")
 
-    canvas = Canvas(json.dumps(cvs.dsl), objs[0].tenant_id)
+    canvas = Canvas(json.dumps(cvs.dsl), objs[0].tenant_id, canvas_id=cvs.id)
     return get_result(
         data={"title": cvs.title, "avatar": cvs.avatar, "inputs": canvas.get_component_input_form("begin"),
               "prologue": canvas.get_prologue(), "mode": canvas.get_mode()})
@@ -918,10 +1012,10 @@ async def ask_about_embedded():
         if search_app := SearchService.get_detail(search_id):
             search_config = search_app.get("search_config", {})
 
-    def stream():
+    async def stream():
         nonlocal req, uid
         try:
-            for ans in ask(req["question"], req["kb_ids"], uid, search_config=search_config):
+            async for ans in async_ask(req["question"], req["kb_ids"], uid, search_config=search_config):
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
         except Exception as e:
             yield "data:" + json.dumps(
@@ -968,25 +1062,26 @@ async def retrieval_test_embedded():
     if not tenant_id:
         return get_error_data_result(message="permission denined.")
 
-    def _retrieval_sync():
+    async def _retrieval():
         local_doc_ids = list(doc_ids) if doc_ids else []
         tenant_ids = []
         _question = question
 
+        meta_data_filter = {}
+        chat_mdl = None
         if req.get("search_id", ""):
             search_config = SearchService.get_detail(req.get("search_id", "")).get("search_config", {})
             meta_data_filter = search_config.get("meta_data_filter", {})
-            metas = DocumentService.get_meta_by_kbs(kb_ids)
-            if meta_data_filter.get("method") == "auto":
+            if meta_data_filter.get("method") in ["auto", "semi_auto"]:
                 chat_mdl = LLMBundle(tenant_id, LLMType.CHAT, llm_name=search_config.get("chat_id", ""))
-                filters: dict = gen_meta_filter(chat_mdl, metas, _question)
-                local_doc_ids.extend(meta_filter(metas, filters["conditions"], filters.get("logic", "and")))
-                if not local_doc_ids:
-                    local_doc_ids = None
-            elif meta_data_filter.get("method") == "manual":
-                local_doc_ids.extend(meta_filter(metas, meta_data_filter["manual"], meta_data_filter.get("logic", "and")))
-                if meta_data_filter["manual"] and not local_doc_ids:
-                    local_doc_ids = ["-999"]
+        else:
+            meta_data_filter = req.get("meta_data_filter") or {}
+            if meta_data_filter.get("method") in ["auto", "semi_auto"]:
+                chat_mdl = LLMBundle(tenant_id, LLMType.CHAT)
+
+        if meta_data_filter:
+            metas = DocumentService.get_meta_by_kbs(kb_ids)
+            local_doc_ids = await apply_meta_data_filter(meta_data_filter, metas, _question, chat_mdl, local_doc_ids)
 
         tenants = UserTenantService.query(user_id=tenant_id)
         for kb_id in kb_ids:
@@ -995,7 +1090,7 @@ async def retrieval_test_embedded():
                     tenant_ids.append(tenant.tenant_id)
                     break
             else:
-                return get_json_result(data=False, message="Only owner of knowledgebase authorized for this operation.",
+                return get_json_result(data=False, message="Only owner of dataset authorized for this operation.",
                                        code=RetCode.OPERATING_ERROR)
 
         e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
@@ -1003,7 +1098,7 @@ async def retrieval_test_embedded():
             return get_error_data_result(message="Knowledgebase not found!")
 
         if langs:
-            _question = cross_languages(kb.tenant_id, None, _question, langs)
+            _question = await cross_languages(kb.tenant_id, None, _question, langs)
 
         embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
 
@@ -1013,7 +1108,7 @@ async def retrieval_test_embedded():
 
         if req.get("keyword", False):
             chat_mdl = LLMBundle(kb.tenant_id, LLMType.CHAT)
-            _question += keyword_extraction(chat_mdl, _question)
+            _question += await keyword_extraction(chat_mdl, _question)
 
         labels = label_question(_question, [kb])
         ranks = settings.retriever.retrieval(
@@ -1033,7 +1128,7 @@ async def retrieval_test_embedded():
         return get_json_result(data=ranks)
 
     try:
-        return await asyncio.to_thread(_retrieval_sync)
+        return await _retrieval()
     except Exception as e:
         if str(e).find("not_found") > 0:
             return get_json_result(data=False, message="No chunk found! Check the chunk status please!",
@@ -1134,7 +1229,7 @@ async def mindmap():
     search_id = req.get("search_id", "")
     search_app = SearchService.get_detail(search_id) if search_id else {}
 
-    mind_map = gen_mindmap(req["question"], req["kb_ids"], tenant_id, search_app.get("search_config", {}))
+    mind_map =await gen_mindmap(req["question"], req["kb_ids"], tenant_id, search_app.get("search_config", {}))
     if "error" in mind_map:
         return server_error_response(Exception(mind_map["error"]))
     return get_json_result(data=mind_map)

api/apps/tenant_app.py

@@ -13,7 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-
+import logging
+import asyncio
 from api.db import UserTenantRole
 from api.db.db_models import UserTenant
 from api.db.services.user_service import UserTenantService, UserService
@@ -24,7 +25,7 @@ from common.time_utils import delta_seconds
 from api.utils.api_utils import get_data_error_result, get_json_result, get_request_json, server_error_response, validate_request
 from api.utils.web_utils import send_invite_email
 from common import settings
-from api.apps import smtp_mail_server, login_required, current_user
+from api.apps import login_required, current_user
 
 
 @manager.route("/<tenant_id>/user/list", methods=["GET"])  # noqa: F821
@@ -80,20 +81,24 @@ async def create(tenant_id):
         role=UserTenantRole.INVITE,
         status=StatusEnum.VALID.value)
 
-    if smtp_mail_server and settings.SMTP_CONF:
-        from threading import Thread
+    try:
 
         user_name = ""
         _, user = UserService.get_by_id(current_user.id)
         if user:
             user_name = user.nickname
 
-        Thread(
-            target=send_invite_email,
-            args=(invite_user_email, settings.MAIL_FRONTEND_URL, tenant_id, user_name or current_user.email),
-            daemon=True
-        ).start()
-
+        asyncio.create_task(
+            send_invite_email(
+                to_email=invite_user_email,
+                invite_url=settings.MAIL_FRONTEND_URL,
+                tenant_id=tenant_id,
+                inviter=user_name or current_user.email
+            )
+        )
+    except Exception as e:
+        logging.exception(f"Failed to send invite email to {invite_user_email}: {e}")
+        return get_json_result(data=False, message="Failed to send invite email.", code=RetCode.SERVER_ERROR)
     usr = invite_users[0].to_dict()
     usr = {k: v for k, v in usr.items() if k in ["id", "avatar", "email", "nickname"]}
 

api/apps/user_app.py

@@ -21,8 +21,9 @@ import re
 import secrets
 import time
 from datetime import datetime
+import base64
 
-from quart import redirect, request, session, make_response
+from quart import make_response, redirect, request, session
 from werkzeug.security import check_password_hash, generate_password_hash
 
 from api.apps.auth import get_auth_client
@@ -45,7 +46,7 @@ from api.utils.api_utils import (
 )
 from api.utils.crypt import decrypt
 from rag.utils.redis_conn import REDIS_CONN
-from api.apps import smtp_mail_server, login_required, current_user, login_user, logout_user
+from api.apps import login_required, current_user, login_user, logout_user
 from api.utils.web_utils import (
     send_email_html,
     OTP_LENGTH,
@@ -97,6 +98,9 @@ async def login():
         return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="Unauthorized!")
 
     email = json_body.get("email", "")
+    if email == "admin@ragflow.io":
+        return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="Default admin account cannot be used to login normal services!")
+    
     users = UserService.query(email=email)
     if not users:
         return get_json_result(
@@ -656,7 +660,7 @@ def user_register(user_id, user):
     tenant_llm = get_init_tenant_llm(user_id)
 
     if not UserService.save(**user):
-        return
+        return None
     TenantService.insert(**tenant)
     UserTenantService.insert(**usr_tenant)
     TenantLLMService.insert_many(tenant_llm)
@@ -923,47 +927,45 @@ async def forget_send_otp():
 
     ttl_min = OTP_TTL_SECONDS // 60
 
-    if not smtp_mail_server:
-        logging.warning("SMTP mail server not initialized; skip sending email.")
-    else:
-        try:
-            send_email_html(
-                subject="Your Password Reset Code",
-                to_email=email,
-                template_key="reset_code",
-                code=otp,
-                ttl_min=ttl_min,
-            )
-        except Exception:
-            return get_json_result(data=False, code=RetCode.SERVER_ERROR, message="failed to send email")
+    try:
+        await send_email_html(
+            subject="Your Password Reset Code",
+            to_email=email,
+            template_key="reset_code",
+            code=otp,
+            ttl_min=ttl_min,
+        )
+
+    except Exception as e:
+        logging.exception(e)
+        return get_json_result(data=False, code=RetCode.SERVER_ERROR, message="failed to send email")
 
     return get_json_result(data=True, code=RetCode.SUCCESS, message="verification passed, email sent")
 
 
-@manager.route("/forget", methods=["POST"])  # noqa: F821
-async def forget():
+def _verified_key(email: str) -> str:
+    return f"otp:verified:{email}"
+
+
+@manager.route("/forget/verify-otp", methods=["POST"])  # noqa: F821
+async def forget_verify_otp():
     """
-    POST: Verify email + OTP and reset password, then log the user in.
-    Request JSON: { email, otp, new_password, confirm_new_password }
+    Verify email + OTP only. On success:
+    - consume the OTP and attempt counters
+    - set a short-lived verified flag in Redis for the email
+    Request JSON: { email, otp }
     """
     req = await get_request_json()
     email = req.get("email") or ""
     otp = (req.get("otp") or "").strip()
-    new_pwd = req.get("new_password")
-    new_pwd2 = req.get("confirm_new_password")
 
-    if not all([email, otp, new_pwd, new_pwd2]):
-        return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="email, otp and passwords are required")
-
-    # For reset, passwords are provided as-is (no decrypt needed)
-    if new_pwd != new_pwd2:
-        return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="passwords do not match")
+    if not all([email, otp]):
+        return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="email and otp are required")
 
     users = UserService.query(email=email)
     if not users:
         return get_json_result(data=False, code=RetCode.DATA_ERROR, message="invalid email")
 
-    user = users[0]
     # Verify OTP from Redis
     k_code, k_attempts, k_last, k_lock = otp_keys(email)
     if REDIS_CONN.get(k_lock):
@@ -979,7 +981,6 @@ async def forget():
     except Exception:
         return get_json_result(data=False, code=RetCode.EXCEPTION_ERROR, message="otp storage corrupted")
 
-    # Case-insensitive verification: OTP generated uppercase
     calc = hash_code(otp.upper(), salt)
     if calc != stored_hash:
         # bump attempts
@@ -992,23 +993,70 @@ async def forget():
             REDIS_CONN.set(k_lock, int(time.time()), ATTEMPT_LOCK_SECONDS)
         return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="expired otp")
 
-    # Success: consume OTP and reset password
+    # Success: consume OTP and attempts; mark verified
     REDIS_CONN.delete(k_code)
     REDIS_CONN.delete(k_attempts)
     REDIS_CONN.delete(k_last)
     REDIS_CONN.delete(k_lock)
 
+    # set verified flag with limited TTL, reuse OTP_TTL_SECONDS or smaller window
     try:
-        UserService.update_user_password(user.id, new_pwd)
+        REDIS_CONN.set(_verified_key(email), "1", OTP_TTL_SECONDS)
+    except Exception:
+        return get_json_result(data=False, code=RetCode.SERVER_ERROR, message="failed to set verification state")
+
+    return get_json_result(data=True, code=RetCode.SUCCESS, message="otp verified")
+
+
+@manager.route("/forget/reset-password", methods=["POST"])  # noqa: F821
+async def forget_reset_password():
+    """
+    Reset password after successful OTP verification.
+    Requires: { email, new_password, confirm_new_password }
+    Steps:
+    - check verified flag in Redis
+    - update user password
+    - auto login
+    - clear verified flag
+    """
+    
+    req = await get_request_json()
+    email = req.get("email") or ""
+    new_pwd = req.get("new_password")
+    new_pwd2 = req.get("confirm_new_password")
+
+    new_pwd_base64 = decrypt(new_pwd)
+    new_pwd_string = base64.b64decode(new_pwd_base64).decode('utf-8')
+    new_pwd2_string = base64.b64decode(decrypt(new_pwd2)).decode('utf-8')
+
+    REDIS_CONN.get(_verified_key(email))
+    if not REDIS_CONN.get(_verified_key(email)):
+        return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="email not verified")
+
+    if not all([email, new_pwd, new_pwd2]):
+        return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="email and passwords are required")
+
+    if new_pwd_string != new_pwd2_string:
+        return get_json_result(data=False, code=RetCode.ARGUMENT_ERROR, message="passwords do not match")
+
+    users = UserService.query_user_by_email(email=email)
+    if not users:
+        return get_json_result(data=False, code=RetCode.DATA_ERROR, message="invalid email")
+    
+    user = users[0]
+    try:
+        UserService.update_user_password(user.id, new_pwd_base64)
     except Exception as e:
         logging.exception(e)
         return get_json_result(data=False, code=RetCode.EXCEPTION_ERROR, message="failed to reset password")
 
-    # Auto login (reuse login flow)
-    user.access_token = get_uuid()
-    login_user(user)
-    user.update_time = current_timestamp()
-    user.update_date = datetime_format(datetime.now())
-    user.save()
+    # clear verified flag
+    try:
+        REDIS_CONN.delete(_verified_key(email))
+    except Exception:
+        pass
+
     msg = "Password reset successful. Logged in."
     return await construct_response(data=user.to_json(), auth=user.get_id(), message=msg)
+
+

api/constants.py

@@ -24,3 +24,5 @@ REQUEST_MAX_WAIT_SEC = 300
 
 DATASET_NAME_LIMIT = 128
 FILE_NAME_LEN_LIMIT = 255
+MEMORY_NAME_LIMIT = 128
+MEMORY_SIZE_LIMIT = 10*1024*1024 # Byte

api/db/db_models.py

@@ -1177,6 +1177,27 @@ class EvaluationResult(DataBaseModel):
         db_table = "evaluation_results"
 
 
+class Memory(DataBaseModel):
+    id = CharField(max_length=32, primary_key=True)
+    name = CharField(max_length=128, null=False, index=False, help_text="Memory name")
+    avatar = TextField(null=True, help_text="avatar base64 string")
+    tenant_id = CharField(max_length=32, null=False, index=True)
+    memory_type = IntegerField(null=False, default=1, index=True, help_text="Bit flags (LSB->MSB): 1=raw, 2=semantic, 4=episodic, 8=procedural. E.g., 5 enables raw + episodic.")
+    storage_type = CharField(max_length=32, default='table', null=False, index=True, help_text="table|graph")
+    embd_id = CharField(max_length=128, null=False, index=False, help_text="embedding model ID")
+    llm_id = CharField(max_length=128, null=False, index=False, help_text="chat model ID")
+    permissions = CharField(max_length=16, null=False, index=True, help_text="me|team", default="me")
+    description = TextField(null=True, help_text="description")
+    memory_size = IntegerField(default=5242880, null=False, index=False)
+    forgetting_policy = CharField(max_length=32, null=False, default="FIFO", index=False, help_text="LRU|FIFO")
+    temperature = FloatField(default=0.5, index=False)
+    system_prompt = TextField(null=True, help_text="system prompt", index=False)
+    user_prompt = TextField(null=True, help_text="user prompt", index=False)
+
+    class Meta:
+        db_table = "memory"
+
+
 def migrate_db():
     logging.disable(logging.ERROR)
     migrator = DatabaseMigrator[settings.DATABASE_TYPE.upper()].value(DB)
@@ -1357,7 +1378,7 @@ def migrate_db():
         migrate(migrator.add_column("llm_factories", "rank", IntegerField(default=0, index=False)))
     except Exception:
         pass
-    
+
     # RAG Evaluation tables
     try:
         migrate(migrator.add_column("evaluation_datasets", "id", CharField(max_length=32, primary_key=True)))
@@ -1395,5 +1416,5 @@ def migrate_db():
         migrate(migrator.add_column("evaluation_datasets", "status", IntegerField(null=False, default=1)))
     except Exception:
         pass
-    
+
     logging.disable(logging.NOTSET)

api/db/init_data.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import asyncio
 import logging
 import json
 import os
@@ -29,6 +30,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.tenant_llm_service import LLMFactoriesService, TenantLLMService
 from api.db.services.llm_service import LLMService, LLMBundle, get_init_tenant_llm
 from api.db.services.user_service import TenantService, UserTenantService
+from api.db.joint_services.memory_message_service import init_message_id_sequence, init_memory_size_cache
 from common.constants import LLMType
 from common.file_utils import get_project_base_directory
 from common import settings
@@ -73,11 +75,10 @@ def init_superuser(nickname=DEFAULT_SUPERUSER_NICKNAME, email=DEFAULT_SUPERUSER_
     UserTenantService.insert(**usr_tenant)
     TenantLLMService.insert_many(tenant_llm)
     logging.info(
-        f"Super user initialized. email: {email}, password: {password}. Changing the password after login is strongly recommended.")
+        f"Super user initialized. email: {email},A default password has been set; changing the password after login is strongly recommended.")
 
     chat_mdl = LLMBundle(tenant["id"], LLMType.CHAT, tenant["llm_id"])
-    msg = chat_mdl.chat(system="", history=[
-        {"role": "user", "content": "Hello!"}], gen_conf={})
+    msg = asyncio.run(chat_mdl.async_chat(system="", history=[{"role": "user", "content": "Hello!"}], gen_conf={}))
     if msg.find("ERROR: ") == 0:
         logging.error(
             "'{}' doesn't work. {}".format(
@@ -169,6 +170,8 @@ def init_web_data():
     #    init_superuser()
 
     add_graph_templates()
+    init_message_id_sequence()
+    init_memory_size_cache()
     logging.info("init web data success:{}".format(time.time() - start_time))
 
 

api/db/joint_services/memory_message_service.py

@@ -0,0 +1,233 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import logging
+from typing import List
+
+from common.time_utils import current_timestamp, timestamp_to_date, format_iso_8601_to_ymd_hms
+from common.constants import MemoryType, LLMType
+from common.doc_store.doc_store_base import FusionExpr
+from api.db.services.memory_service import MemoryService
+from api.db.services.tenant_llm_service import TenantLLMService
+from api.db.services.llm_service import LLMBundle
+from api.utils.memory_utils import get_memory_type_human
+from memory.services.messages import MessageService
+from memory.services.query import MsgTextQuery, get_vector
+from memory.utils.prompt_util import PromptAssembler
+from memory.utils.msg_util import get_json_result_from_llm_response
+from rag.utils.redis_conn import REDIS_CONN
+
+
+async def save_to_memory(memory_id: str, message_dict: dict):
+    """
+    :param memory_id:
+    :param message_dict: {
+        "user_id": str,
+        "agent_id": str,
+        "session_id": str,
+        "user_input": str,
+        "agent_response": str
+    }
+    """
+    memory = MemoryService.get_by_memory_id(memory_id)
+    if not memory:
+        return False, f"Memory '{memory_id}' not found."
+
+    tenant_id = memory.tenant_id
+    extracted_content = await extract_by_llm(
+        tenant_id,
+        memory.llm_id,
+        {"temperature": memory.temperature},
+        get_memory_type_human(memory.memory_type),
+        message_dict.get("user_input", ""),
+        message_dict.get("agent_response", "")
+    ) if memory.memory_type != MemoryType.RAW.value else []  # if only RAW, no need to extract
+    raw_message_id = REDIS_CONN.generate_auto_increment_id(namespace="memory")
+    message_list = [{
+        "message_id": raw_message_id,
+        "message_type": MemoryType.RAW.name.lower(),
+        "source_id": 0,
+        "memory_id": memory_id,
+        "user_id": "",
+        "agent_id": message_dict["agent_id"],
+        "session_id": message_dict["session_id"],
+        "content": f"User Input: {message_dict.get('user_input')}\nAgent Response: {message_dict.get('agent_response')}",
+        "valid_at": timestamp_to_date(current_timestamp()),
+        "invalid_at": None,
+        "forget_at": None,
+        "status": True
+    }, *[{
+        "message_id": REDIS_CONN.generate_auto_increment_id(namespace="memory"),
+        "message_type": content["message_type"],
+        "source_id": raw_message_id,
+        "memory_id": memory_id,
+        "user_id": "",
+        "agent_id": message_dict["agent_id"],
+        "session_id": message_dict["session_id"],
+        "content": content["content"],
+        "valid_at": content["valid_at"],
+        "invalid_at": content["invalid_at"] if content["invalid_at"] else None,
+        "forget_at": None,
+        "status": True
+    } for content in extracted_content]]
+    embedding_model = LLMBundle(tenant_id, llm_type=LLMType.EMBEDDING, llm_name=memory.embd_id)
+    vector_list, _ = embedding_model.encode([msg["content"] for msg in message_list])
+    for idx, msg in enumerate(message_list):
+        msg["content_embed"] = vector_list[idx]
+    vector_dimension = len(vector_list[0])
+    if not MessageService.has_index(tenant_id, memory_id):
+        created = MessageService.create_index(tenant_id, memory_id, vector_size=vector_dimension)
+        if not created:
+            return False, "Failed to create message index."
+
+    new_msg_size = sum([MessageService.calculate_message_size(m) for m in message_list])
+    current_memory_size = get_memory_size_cache(memory_id, tenant_id)
+    if new_msg_size + current_memory_size > memory.memory_size:
+        size_to_delete = current_memory_size + new_msg_size - memory.memory_size
+        if memory.forgetting_policy == "FIFO":
+            message_ids_to_delete, delete_size = MessageService.pick_messages_to_delete_by_fifo(memory_id, tenant_id, size_to_delete)
+            MessageService.delete_message({"message_id": message_ids_to_delete}, tenant_id, memory_id)
+            decrease_memory_size_cache(memory_id, delete_size)
+        else:
+            return False, "Failed to insert message into memory. Memory size reached limit and cannot decide which to delete."
+    fail_cases = MessageService.insert_message(message_list, tenant_id, memory_id)
+    if fail_cases:
+        return False, "Failed to insert message into memory. Details: " + "; ".join(fail_cases)
+
+    increase_memory_size_cache(memory_id, new_msg_size)
+    return True, "Message saved successfully."
+
+
+async def extract_by_llm(tenant_id: str, llm_id: str, extract_conf: dict, memory_type: List[str], user_input: str,
+                         agent_response: str, system_prompt: str = "", user_prompt: str="") -> List[dict]:
+    llm_type = TenantLLMService.llm_id2llm_type(llm_id)
+    if not llm_type:
+        raise RuntimeError(f"Unknown type of LLM '{llm_id}'")
+    if not system_prompt:
+        system_prompt = PromptAssembler.assemble_system_prompt({"memory_type": memory_type})
+    conversation_content = f"User Input: {user_input}\nAgent Response: {agent_response}"
+    conversation_time = timestamp_to_date(current_timestamp())
+    user_prompts = []
+    if user_prompt:
+        user_prompts.append({"role": "user", "content": user_prompt})
+        user_prompts.append({"role": "user", "content": f"Conversation: {conversation_content}\nConversation Time: {conversation_time}\nCurrent Time: {conversation_time}"})
+    else:
+        user_prompts.append({"role": "user", "content": PromptAssembler.assemble_user_prompt(conversation_content, conversation_time, conversation_time)})
+    llm = LLMBundle(tenant_id, llm_type, llm_id)
+    res = await llm.async_chat(system_prompt, user_prompts, extract_conf)
+    res_json = get_json_result_from_llm_response(res)
+    return [{
+        "content": extracted_content["content"],
+        "valid_at": format_iso_8601_to_ymd_hms(extracted_content["valid_at"]),
+        "invalid_at": format_iso_8601_to_ymd_hms(extracted_content["invalid_at"]) if extracted_content.get("invalid_at") else "",
+        "message_type": message_type
+    } for message_type, extracted_content_list in res_json.items() for extracted_content in extracted_content_list]
+
+
+def query_message(filter_dict: dict, params: dict):
+    """
+    :param filter_dict: {
+        "memory_id": List[str],
+        "agent_id": optional
+        "session_id": optional
+    }
+    :param params: {
+        "query": question str,
+        "similarity_threshold": float,
+        "keywords_similarity_weight": float,
+        "top_n": int
+    }
+    """
+    memory_ids = filter_dict["memory_id"]
+    memory_list = MemoryService.get_by_ids(memory_ids)
+    if not memory_list:
+        return []
+
+    condition_dict = {k: v for k, v in filter_dict.items() if v}
+    uids = [memory.tenant_id for memory in memory_list]
+
+    question = params["query"]
+    question = question.strip()
+    memory = memory_list[0]
+    embd_model = LLMBundle(memory.tenant_id, llm_type=LLMType.EMBEDDING, llm_name=memory.embd_id)
+    match_dense = get_vector(question, embd_model, similarity=params["similarity_threshold"])
+    match_text, _ = MsgTextQuery().question(question, min_match=params["similarity_threshold"])
+    keywords_similarity_weight = params.get("keywords_similarity_weight", 0.7)
+    fusion_expr = FusionExpr("weighted_sum", params["top_n"], {"weights": ",".join([str(1 - keywords_similarity_weight), str(keywords_similarity_weight)])})
+
+    return MessageService.search_message(memory_ids, condition_dict, uids, [match_text, match_dense, fusion_expr], params["top_n"])
+
+
+def init_message_id_sequence():
+    message_id_redis_key = "id_generator:memory"
+    if REDIS_CONN.exist(message_id_redis_key):
+        current_max_id = REDIS_CONN.get(message_id_redis_key)
+        logging.info(f"No need to init message_id sequence, current max id is {current_max_id}.")
+    else:
+        max_id = 1
+        exist_memory_list = MemoryService.get_all_memory()
+        if not exist_memory_list:
+            REDIS_CONN.set(message_id_redis_key, max_id)
+        else:
+            max_id = MessageService.get_max_message_id(
+                uid_list=[m.tenant_id for m in exist_memory_list],
+                memory_ids=[m.id for m in exist_memory_list]
+            )
+            REDIS_CONN.set(message_id_redis_key, max_id)
+        logging.info(f"Init message_id sequence done, current max id is {max_id}.")
+
+
+def get_memory_size_cache(memory_id: str, uid: str):
+    redis_key = f"memory_{memory_id}"
+    if REDIS_CONN.exist(redis_key):
+        return int(REDIS_CONN.get(redis_key))
+    else:
+        memory_size_map = MessageService.calculate_memory_size(
+            [memory_id],
+            [uid]
+        )
+        memory_size = memory_size_map.get(memory_id, 0)
+        set_memory_size_cache(memory_id, memory_size)
+        return memory_size
+
+
+def set_memory_size_cache(memory_id: str, size: int):
+    redis_key = f"memory_{memory_id}"
+    return REDIS_CONN.set(redis_key, size)
+
+
+def increase_memory_size_cache(memory_id: str, size: int):
+    redis_key = f"memory_{memory_id}"
+    return REDIS_CONN.incrby(redis_key, size)
+
+
+def decrease_memory_size_cache(memory_id: str, size: int):
+    redis_key = f"memory_{memory_id}"
+    return REDIS_CONN.decrby(redis_key, size)
+
+
+def init_memory_size_cache():
+    memory_list = MemoryService.get_all_memory()
+    if not memory_list:
+        logging.info("No memory found, no need to init memory size.")
+    else:
+        for m in memory_list:
+            get_memory_size_cache(m.id, m.tenant_id)
+        logging.info("Memory size cache init done.")
+
+
+def judge_system_prompt_is_default(system_prompt: str, memory_type: int|list[str]):
+    memory_type_list = memory_type if isinstance(memory_type, list) else get_memory_type_human(memory_type)
+    return system_prompt == PromptAssembler.assemble_system_prompt({"memory_type": memory_type_list})

api/db/joint_services/user_account_service.py

@@ -34,6 +34,8 @@ from api.db.services.task_service import TaskService
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_canvas_version import UserCanvasVersionService
 from api.db.services.user_service import TenantService, UserService, UserTenantService
+from api.db.services.memory_service import MemoryService
+from memory.services.messages import MessageService
 from rag.nlp import search
 from common.constants import ActiveEnum
 from common import settings
@@ -153,7 +155,7 @@ def delete_user_data(user_id: str) -> dict:
             done_msg += "Start to delete owned tenant.\n"
             tenant_id = owned_tenant[0]["tenant_id"]
             kb_ids = KnowledgebaseService.get_kb_ids(usr.id)
-            # step1.1 delete knowledgebase related file and info
+            # step1.1 delete dataset related file and info
             if kb_ids:
                 # step1.1.1 delete files in storage, remove bucket
                 for kb_id in kb_ids:
@@ -182,7 +184,7 @@ def delete_user_data(user_id: str) -> dict:
                                          search.index_name(tenant_id), kb_ids)
                 done_msg += f"- Deleted {r} chunk records.\n"
                 kb_delete_res = KnowledgebaseService.delete_by_ids(kb_ids)
-                done_msg += f"- Deleted {kb_delete_res} knowledgebase records.\n"
+                done_msg += f"- Deleted {kb_delete_res} dataset records.\n"
                 # step1.1.4 delete agents
                 agent_delete_res = delete_user_agents(usr.id)
                 done_msg += f"- Deleted {agent_delete_res['agents_deleted_count']} agent, {agent_delete_res['version_deleted_count']} versions records.\n"
@@ -200,7 +202,16 @@ def delete_user_data(user_id: str) -> dict:
             done_msg += f"- Deleted {llm_delete_res} tenant-LLM records.\n"
             langfuse_delete_res = TenantLangfuseService.delete_ty_tenant_id(tenant_id)
             done_msg += f"- Deleted {langfuse_delete_res} langfuse records.\n"
-            # step1.3 delete own tenant
+            # step1.3 delete memory and messages
+            user_memory = MemoryService.get_by_tenant_id(tenant_id)
+            if user_memory:
+                for memory in user_memory:
+                    if MessageService.has_index(tenant_id, memory.id):
+                        MessageService.delete_index(tenant_id, memory.id)
+                done_msg += " Deleted memory index."
+                memory_delete_res = MemoryService.delete_by_ids([m.id for m in user_memory])
+                done_msg += f"Deleted {memory_delete_res} memory datasets."
+            # step1.4 delete own tenant
             tenant_delete_res = TenantService.delete_by_id(tenant_id)
             done_msg += f"- Deleted {tenant_delete_res} tenant.\n"
         # step2 delete user-tenant relation
@@ -258,7 +269,7 @@ def delete_user_data(user_id: str) -> dict:
                     # step2.1.5 delete document record
                     doc_delete_res = DocumentService.delete_by_ids([d['id'] for d in created_documents])
                     done_msg += f"- Deleted {doc_delete_res} documents.\n"
-                    # step2.1.6 update knowledge base doc&chunk&token cnt
+                    # step2.1.6 update dataset doc&chunk&token cnt
                     for kb_id, doc_num in kb_doc_info.items():
                         KnowledgebaseService.decrease_document_num_in_delete(kb_id, doc_num)
 
@@ -273,7 +284,7 @@ def delete_user_data(user_id: str) -> dict:
 
     except Exception as e:
         logging.exception(e)
-        return {"success": False, "message": f"Error: {str(e)}. Already done:\n{done_msg}"}
+        return {"success": False, "message": "An internal error occurred during user deletion. Some operations may have completed.","details": done_msg}
 
 
 def delete_user_agents(user_id: str) -> dict:

api/db/services/canvas_service.py

@@ -123,6 +123,19 @@ class UserCanvasService(CommonService):
             logging.exception(e)
             return False, None
 
+    @classmethod
+    @DB.connection_context()
+    def get_basic_info_by_canvas_ids(cls, canvas_id):
+        fields = [
+            cls.model.id,
+            cls.model.avatar,
+            cls.model.user_id,
+            cls.model.title,
+            cls.model.permission,
+            cls.model.canvas_category
+        ]
+        return cls.model.select(*fields).where(cls.model.id.in_(canvas_id)).dicts()
+
     @classmethod
     @DB.connection_context()
     def get_by_tenant_ids(cls, joined_tenant_ids, user_id,
@@ -198,7 +211,7 @@ async def completion(tenant_id, agent_id, session_id=None, **kwargs):
         if not isinstance(cvs.dsl, str):
             cvs.dsl = json.dumps(cvs.dsl, ensure_ascii=False)
         session_id=get_uuid()
-        canvas = Canvas(cvs.dsl, tenant_id, agent_id)
+        canvas = Canvas(cvs.dsl, tenant_id, agent_id, canvas_id=cvs.id)
         canvas.reset()
         conv = {
             "id": session_id,

api/db/services/common_service.py

@@ -169,10 +169,12 @@ class CommonService:
         """
         if "id" not in kwargs:
             kwargs["id"] = get_uuid()
-        kwargs["create_time"] = current_timestamp()
-        kwargs["create_date"] = datetime_format(datetime.now())
-        kwargs["update_time"] = current_timestamp()
-        kwargs["update_date"] = datetime_format(datetime.now())
+        timestamp = current_timestamp()
+        cur_datetime = datetime_format(datetime.now())
+        kwargs["create_time"] = timestamp
+        kwargs["create_date"] = cur_datetime
+        kwargs["update_time"] = timestamp
+        kwargs["update_date"] = cur_datetime
         sample_obj = cls.model(**kwargs).save(force_insert=True)
         return sample_obj
 
@@ -207,10 +209,14 @@ class CommonService:
             data_list (list): List of dictionaries containing record data to update.
                              Each dictionary must include an 'id' field.
         """
+
+        timestamp = current_timestamp()
+        cur_datetime = datetime_format(datetime.now())
+        for data in data_list:
+            data["update_time"] = timestamp
+            data["update_date"] = cur_datetime
         with DB.atomic():
             for data in data_list:
-                data["update_time"] = current_timestamp()
-                data["update_date"] = datetime_format(datetime.now())
                 cls.model.update(data).where(cls.model.id == data["id"]).execute()
 
     @classmethod

api/db/services/conversation_service.py

@@ -19,7 +19,7 @@ from common.constants import StatusEnum
 from api.db.db_models import Conversation, DB
 from api.db.services.api_service import API4ConversationService
 from api.db.services.common_service import CommonService
-from api.db.services.dialog_service import DialogService, chat
+from api.db.services.dialog_service import DialogService, async_chat
 from common.misc_utils import get_uuid
 import json
 
@@ -89,8 +89,7 @@ def structure_answer(conv, ans, message_id, session_id):
         conv.reference[-1] = reference
     return ans
 
-
-def completion(tenant_id, chat_id, question, name="New session", session_id=None, stream=True, **kwargs):
+async def async_completion(tenant_id, chat_id, question, name="New session", session_id=None, stream=True, **kwargs):
     assert name, "`name` can not be empty."
     dia = DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value)
     assert dia, "You do not own the chat."
@@ -112,11 +111,21 @@ def completion(tenant_id, chat_id, question, name="New session", session_id=None
                                             "reference": {},
                                             "audio_binary": None,
                                             "id": None,
-                                            "session_id": session_id
+                                        "session_id": session_id
                                         }},
                                     ensure_ascii=False) + "\n\n"
             yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
             return
+        else:
+            answer = {
+                "answer": conv["message"][0]["content"],
+                "reference": {},
+                "audio_binary": None,
+                "id": None,
+                "session_id": session_id
+            }
+            yield answer
+            return
 
     conv = ConversationService.query(id=session_id, dialog_id=chat_id)
     if not conv:
@@ -148,7 +157,7 @@ def completion(tenant_id, chat_id, question, name="New session", session_id=None
 
     if stream:
         try:
-            for ans in chat(dia, msg, True, **kwargs):
+            async for ans in async_chat(dia, msg, True, **kwargs):
                 ans = structure_answer(conv, ans, message_id, session_id)
                 yield "data:" + json.dumps({"code": 0, "data": ans}, ensure_ascii=False) + "\n\n"
             ConversationService.update_by_id(conv.id, conv.to_dict())
@@ -160,14 +169,13 @@ def completion(tenant_id, chat_id, question, name="New session", session_id=None
 
     else:
         answer = None
-        for ans in chat(dia, msg, False, **kwargs):
+        async for ans in async_chat(dia, msg, False, **kwargs):
             answer = structure_answer(conv, ans, message_id, session_id)
             ConversationService.update_by_id(conv.id, conv.to_dict())
             break
         yield answer
 
-
-def iframe_completion(dialog_id, question, session_id=None, stream=True, **kwargs):
+async def async_iframe_completion(dialog_id, question, session_id=None, stream=True, **kwargs):
     e, dia = DialogService.get_by_id(dialog_id)
     assert e, "Dialog not found"
     if not session_id:
@@ -222,7 +230,7 @@ def iframe_completion(dialog_id, question, session_id=None, stream=True, **kwarg
 
     if stream:
         try:
-            for ans in chat(dia, msg, True, **kwargs):
+            async for ans in async_chat(dia, msg, True, **kwargs):
                 ans = structure_answer(conv, ans, message_id, session_id)
                 yield "data:" + json.dumps({"code": 0, "message": "", "data": ans},
                                            ensure_ascii=False) + "\n\n"
@@ -235,7 +243,7 @@ def iframe_completion(dialog_id, question, session_id=None, stream=True, **kwarg
 
     else:
         answer = None
-        for ans in chat(dia, msg, False, **kwargs):
+        async for ans in async_chat(dia, msg, False, **kwargs):
             answer = structure_answer(conv, ans, message_id, session_id)
             API4ConversationService.append_message(conv.id, conv.to_dict())
             break

api/db/services/dialog_service.py

@@ -21,7 +21,6 @@ from copy import deepcopy
 from datetime import datetime
 from functools import partial
 from timeit import default_timer as timer
-import trio
 from langfuse import Langfuse
 from peewee import fn
 from agentic_reasoning import DeepResearcher
@@ -33,6 +32,7 @@ from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.langfuse_service import TenantLangfuseService
 from api.db.services.llm_service import LLMBundle
+from common.metadata_utils import apply_meta_data_filter
 from api.db.services.tenant_llm_service import TenantLLMService
 from common.time_utils import current_timestamp, datetime_format
 from graphrag.general.mind_map_extractor import MindMapExtractor
@@ -40,7 +40,7 @@ from rag.app.resume import forbidden_select_fields4resume
 from rag.app.tag import label_question
 from rag.nlp.search import index_name
 from rag.prompts.generator import chunks_format, citation_prompt, cross_languages, full_question, kb_prompt, keyword_extraction, message_fit_in, \
-    gen_meta_filter, PROMPT_JINJA_ENV, ASK_SUMMARY
+    PROMPT_JINJA_ENV, ASK_SUMMARY
 from common.token_utils import num_tokens_from_string
 from rag.utils.tavily_conn import Tavily
 from common.string_utils import remove_redundant_spaces
@@ -178,7 +178,8 @@ class DialogService(CommonService):
             offset += limit
         return res
 
-def chat_solo(dialog, messages, stream=True):
+
+async def async_chat_solo(dialog, messages, stream=True):
     attachments = ""
     if "files" in messages[-1]:
         attachments = "\n\n".join(FileService.get_files(messages[-1]["files"]))
@@ -197,7 +198,8 @@ def chat_solo(dialog, messages, stream=True):
     if stream:
         last_ans = ""
         delta_ans = ""
-        for ans in chat_mdl.chat_streamly(prompt_config.get("system", ""), msg, dialog.llm_setting):
+        answer = ""
+        async for ans in chat_mdl.async_chat_streamly(prompt_config.get("system", ""), msg, dialog.llm_setting):
             answer = ans
             delta_ans = ans[len(last_ans):]
             if num_tokens_from_string(delta_ans) < 16:
@@ -208,7 +210,7 @@ def chat_solo(dialog, messages, stream=True):
         if delta_ans:
             yield {"answer": answer, "reference": {}, "audio_binary": tts(tts_mdl, delta_ans), "prompt": "", "created_at": time.time()}
     else:
-        answer = chat_mdl.chat(prompt_config.get("system", ""), msg, dialog.llm_setting)
+        answer = await chat_mdl.async_chat(prompt_config.get("system", ""), msg, dialog.llm_setting)
         user_content = msg[-1].get("content", "[content not available]")
         logging.debug("User: {}|Assistant: {}".format(user_content, answer))
         yield {"answer": answer, "reference": {}, "audio_binary": tts(tts_mdl, answer), "prompt": "", "created_at": time.time()}
@@ -276,84 +278,12 @@ def repair_bad_citation_formats(answer: str, kbinfos: dict, idx: set):
     return answer, idx
 
 
-def convert_conditions(metadata_condition):
-    if metadata_condition is None:
-        metadata_condition = {}
-    op_mapping = {
-        "is": "=",
-        "not is": "≠"
-    }
-    return [
-        {
-            "op": op_mapping.get(cond["comparison_operator"], cond["comparison_operator"]),
-            "key": cond["name"],
-            "value": cond["value"]
-        }
-        for cond in metadata_condition.get("conditions", [])
-    ]
-
-
-def meta_filter(metas: dict, filters: list[dict], logic: str = "and"):
-    doc_ids = set([])
-
-    def filter_out(v2docs, operator, value):
-        ids = []
-        for input, docids in v2docs.items():
-            if operator in ["=", "≠", ">", "<", "≥", "≤"]:
-                try:
-                    input = float(input)
-                    value = float(value)
-                except Exception:
-                    input = str(input)
-                    value = str(value)
-
-            for conds in [
-                (operator == "contains", str(value).lower() in str(input).lower()),
-                (operator == "not contains", str(value).lower() not in str(input).lower()),
-                (operator == "in", str(input).lower() in str(value).lower()),
-                (operator == "not in", str(input).lower() not in str(value).lower()),
-                (operator == "start with", str(input).lower().startswith(str(value).lower())),
-                (operator == "end with", str(input).lower().endswith(str(value).lower())),
-                (operator == "empty", not input),
-                (operator == "not empty", input),
-                (operator == "=", input == value),
-                (operator == "≠", input != value),
-                (operator == ">", input > value),
-                (operator == "<", input < value),
-                (operator == "≥", input >= value),
-                (operator == "≤", input <= value),
-            ]:
-                try:
-                    if all(conds):
-                        ids.extend(docids)
-                        break
-                except Exception:
-                    pass
-        return ids
-
-    for k, v2docs in metas.items():
-        for f in filters:
-            if k != f["key"]:
-                continue
-            ids = filter_out(v2docs, f["op"], f["value"])
-            if not doc_ids:
-                doc_ids = set(ids)
-            else:
-                if logic == "and":
-                    doc_ids = doc_ids & set(ids)
-                else:
-                    doc_ids = doc_ids | set(ids)
-            if not doc_ids:
-                return []
-    return list(doc_ids)
-
-
-def chat(dialog, messages, stream=True, **kwargs):
+async def async_chat(dialog, messages, stream=True, **kwargs):
     assert messages[-1]["role"] == "user", "The last content of this conversation is not from user."
     if not dialog.kb_ids and not dialog.prompt_config.get("tavily_api_key"):
-        for ans in chat_solo(dialog, messages, stream):
+        async for ans in async_chat_solo(dialog, messages, stream):
             yield ans
-        return None
+        return
 
     chat_start_ts = timer()
 
@@ -397,10 +327,10 @@ def chat(dialog, messages, stream=True, **kwargs):
     # try to use sql if field mapping is good to go
     if field_map:
         logging.debug("Use SQL to retrieval:{}".format(questions[-1]))
-        ans = use_sql(questions[-1], field_map, dialog.tenant_id, chat_mdl, prompt_config.get("quote", True), dialog.kb_ids)
+        ans = await use_sql(questions[-1], field_map, dialog.tenant_id, chat_mdl, prompt_config.get("quote", True), dialog.kb_ids)
         if ans:
             yield ans
-            return None
+            return
 
     for p in prompt_config["parameters"]:
         if p["key"] == "knowledge":
@@ -411,28 +341,25 @@ def chat(dialog, messages, stream=True, **kwargs):
             prompt_config["system"] = prompt_config["system"].replace("{%s}" % p["key"], " ")
 
     if len(questions) > 1 and prompt_config.get("refine_multiturn"):
-        questions = [full_question(dialog.tenant_id, dialog.llm_id, messages)]
+        questions = [await full_question(dialog.tenant_id, dialog.llm_id, messages)]
     else:
         questions = questions[-1:]
 
     if prompt_config.get("cross_languages"):
-        questions = [cross_languages(dialog.tenant_id, dialog.llm_id, questions[0], prompt_config["cross_languages"])]
+        questions = [await cross_languages(dialog.tenant_id, dialog.llm_id, questions[0], prompt_config["cross_languages"])]
 
     if dialog.meta_data_filter:
         metas = DocumentService.get_meta_by_kbs(dialog.kb_ids)
-        if dialog.meta_data_filter.get("method") == "auto":
-            filters: dict = gen_meta_filter(chat_mdl, metas, questions[-1])
-            attachments.extend(meta_filter(metas, filters["conditions"], filters.get("logic", "and")))
-            if not attachments:
-                attachments = None
-        elif dialog.meta_data_filter.get("method") == "manual":
-            conds = dialog.meta_data_filter["manual"]
-            attachments.extend(meta_filter(metas, conds, dialog.meta_data_filter.get("logic", "and")))
-            if conds and not attachments:
-                attachments = ["-999"]
+        attachments = await apply_meta_data_filter(
+            dialog.meta_data_filter,
+            metas,
+            questions[-1],
+            chat_mdl,
+            attachments,
+        )
 
     if prompt_config.get("keyword", False):
-        questions[-1] += keyword_extraction(chat_mdl, questions[-1])
+        questions[-1] += await keyword_extraction(chat_mdl, questions[-1])
 
     refine_question_ts = timer()
 
@@ -460,7 +387,7 @@ def chat(dialog, messages, stream=True, **kwargs):
                 ),
             )
 
-            for think in reasoner.thinking(kbinfos, attachments_ + " ".join(questions)):
+            async for think in reasoner.thinking(kbinfos, attachments_ + " ".join(questions)):
                 if isinstance(think, str):
                     thought = think
                     knowledges = [t for t in think.split("\n") if t]
@@ -479,7 +406,7 @@ def chat(dialog, messages, stream=True, **kwargs):
                     dialog.vector_similarity_weight,
                     doc_ids=attachments,
                     top=dialog.top_k,
-                    aggs=False,
+                    aggs=True,
                     rerank_mdl=rerank_mdl,
                     rank_feature=label_question(" ".join(questions), kbs),
                 )
@@ -508,7 +435,8 @@ def chat(dialog, messages, stream=True, **kwargs):
         empty_res = prompt_config["empty_response"]
         yield {"answer": empty_res, "reference": kbinfos, "prompt": "\n\n### Query:\n%s" % " ".join(questions),
                "audio_binary": tts(tts_mdl, empty_res)}
-        return {"answer": prompt_config["empty_response"], "reference": kbinfos}
+        yield {"answer": prompt_config["empty_response"], "reference": kbinfos}
+        return
 
     kwargs["knowledge"] = "\n------\n" + "\n\n------\n\n".join(knowledges)
     gen_conf = dialog.llm_setting
@@ -612,7 +540,7 @@ def chat(dialog, messages, stream=True, **kwargs):
     if stream:
         last_ans = ""
         answer = ""
-        for ans in chat_mdl.chat_streamly(prompt + prompt4citation, msg[1:], gen_conf):
+        async for ans in chat_mdl.async_chat_streamly(prompt + prompt4citation, msg[1:], gen_conf):
             if thought:
                 ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
             answer = ans
@@ -626,19 +554,19 @@ def chat(dialog, messages, stream=True, **kwargs):
             yield {"answer": thought + answer, "reference": {}, "audio_binary": tts(tts_mdl, delta_ans)}
         yield decorate_answer(thought + answer)
     else:
-        answer = chat_mdl.chat(prompt + prompt4citation, msg[1:], gen_conf)
+        answer = await chat_mdl.async_chat(prompt + prompt4citation, msg[1:], gen_conf)
         user_content = msg[-1].get("content", "[content not available]")
         logging.debug("User: {}|Assistant: {}".format(user_content, answer))
         res = decorate_answer(answer)
         res["audio_binary"] = tts(tts_mdl, answer)
         yield res
 
-    return None
+    return
 
 
-def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=None):
+async def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=None):
     sys_prompt = """
-You are a Database Administrator. You need to check the fields of the following tables based on the user's list of questions and write the SQL corresponding to the last question. 
+You are a Database Administrator. You need to check the fields of the following tables based on the user's list of questions and write the SQL corresponding to the last question.
 Ensure that:
 1. Field names should not start with a digit. If any field name starts with a digit, use double quotes around it.
 2. Write only the SQL, no explanations or additional text.
@@ -654,9 +582,9 @@ Please write the SQL, only SQL, without any other explanations or text.
 """.format(index_name(tenant_id), "\n".join([f"{k}: {v}" for k, v in field_map.items()]), question)
     tried_times = 0
 
-    def get_table():
+    async def get_table():
         nonlocal sys_prompt, user_prompt, question, tried_times
-        sql = chat_mdl.chat(sys_prompt, [{"role": "user", "content": user_prompt}], {"temperature": 0.06})
+        sql = await chat_mdl.async_chat(sys_prompt, [{"role": "user", "content": user_prompt}], {"temperature": 0.06})
         sql = re.sub(r"^.*</think>", "", sql, flags=re.DOTALL)
         logging.debug(f"{question} ==> {user_prompt} get SQL: {sql}")
         sql = re.sub(r"[\r\n]+", " ", sql.lower())
@@ -695,7 +623,7 @@ Please write the SQL, only SQL, without any other explanations or text.
         return settings.retriever.sql_retrieval(sql, format="json"), sql
 
     try:
-        tbl, sql = get_table()
+        tbl, sql = await get_table()
     except Exception as e:
         user_prompt = """
         Table name: {};
@@ -713,7 +641,7 @@ Please write the SQL, only SQL, without any other explanations or text.
         Please correct the error and write SQL again, only SQL, without any other explanations or text.
         """.format(index_name(tenant_id), "\n".join([f"{k}: {v}" for k, v in field_map.items()]), question, e)
         try:
-            tbl, sql = get_table()
+            tbl, sql = await get_table()
         except Exception:
             return
 
@@ -805,8 +733,7 @@ def tts(tts_mdl, text):
         return None
     return binascii.hexlify(bin).decode("utf-8")
 
-
-def ask(question, kb_ids, tenant_id, chat_llm_name=None, search_config={}):
+async def async_ask(question, kb_ids, tenant_id, chat_llm_name=None, search_config={}):
     doc_ids = search_config.get("doc_ids", [])
     rerank_mdl = None
     kb_ids = search_config.get("kb_ids", kb_ids)
@@ -829,15 +756,7 @@ def ask(question, kb_ids, tenant_id, chat_llm_name=None, search_config={}):
 
     if meta_data_filter:
         metas = DocumentService.get_meta_by_kbs(kb_ids)
-        if meta_data_filter.get("method") == "auto":
-            filters: dict = gen_meta_filter(chat_mdl, metas, question)
-            doc_ids.extend(meta_filter(metas, filters["conditions"], filters.get("logic", "and")))
-            if not doc_ids:
-                doc_ids = None
-        elif meta_data_filter.get("method") == "manual":
-            doc_ids.extend(meta_filter(metas, meta_data_filter["manual"], meta_data_filter.get("logic", "and")))
-            if meta_data_filter["manual"] and not doc_ids:
-                doc_ids = ["-999"]
+        doc_ids = await apply_meta_data_filter(meta_data_filter, metas, question, chat_mdl, doc_ids)
 
     kbinfos = retriever.retrieval(
         question=question,
@@ -850,7 +769,7 @@ def ask(question, kb_ids, tenant_id, chat_llm_name=None, search_config={}):
         vector_similarity_weight=search_config.get("vector_similarity_weight", 0.3),
         top=search_config.get("top_k", 1024),
         doc_ids=doc_ids,
-        aggs=False,
+        aggs=True,
         rerank_mdl=rerank_mdl,
         rank_feature=label_question(question, kbs)
     )
@@ -880,13 +799,13 @@ def ask(question, kb_ids, tenant_id, chat_llm_name=None, search_config={}):
         return {"answer": answer, "reference": refs}
 
     answer = ""
-    for ans in chat_mdl.chat_streamly(sys_prompt, msg, {"temperature": 0.1}):
+    async for ans in chat_mdl.async_chat_streamly(sys_prompt, msg, {"temperature": 0.1}):
         answer = ans
         yield {"answer": answer, "reference": {}}
     yield decorate_answer(answer)
 
 
-def gen_mindmap(question, kb_ids, tenant_id, search_config={}):
+async def gen_mindmap(question, kb_ids, tenant_id, search_config={}):
     meta_data_filter = search_config.get("meta_data_filter", {})
     doc_ids = search_config.get("doc_ids", [])
     rerank_id = search_config.get("rerank_id", "")
@@ -904,15 +823,7 @@ def gen_mindmap(question, kb_ids, tenant_id, search_config={}):
 
     if meta_data_filter:
         metas = DocumentService.get_meta_by_kbs(kb_ids)
-        if meta_data_filter.get("method") == "auto":
-            filters: dict = gen_meta_filter(chat_mdl, metas, question)
-            doc_ids.extend(meta_filter(metas, filters["conditions"], filters.get("logic", "and")))
-            if not doc_ids:
-                doc_ids = None
-        elif meta_data_filter.get("method") == "manual":
-            doc_ids.extend(meta_filter(metas, meta_data_filter["manual"], meta_data_filter.get("logic", "and")))
-            if meta_data_filter["manual"] and not doc_ids:
-                doc_ids = ["-999"]
+        doc_ids = await apply_meta_data_filter(meta_data_filter, metas, question, chat_mdl, doc_ids)
 
     ranks = settings.retriever.retrieval(
         question=question,
@@ -930,5 +841,5 @@ def gen_mindmap(question, kb_ids, tenant_id, search_config={}):
         rank_feature=label_question(question, kbs),
     )
     mindmap = MindMapExtractor(chat_mdl)
-    mind_map = trio.run(mindmap, [c["content_with_weight"] for c in ranks["chunks"]])
+    mind_map = await mindmap([c["content_with_weight"] for c in ranks["chunks"]])
     return mind_map.output

api/db/services/document_service.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import asyncio
 import json
 import logging
 import random
@@ -22,7 +23,6 @@ from copy import deepcopy
 from datetime import datetime
 from io import BytesIO
 
-import trio
 import xxhash
 from peewee import fn, Case, JOIN
 
@@ -33,12 +33,13 @@ from api.db.db_models import DB, Document, Knowledgebase, Task, Tenant, UserTena
 from api.db.db_utils import bulk_insert_into_db
 from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
+from common.metadata_utils import dedupe_list
 from common.misc_utils import get_uuid
 from common.time_utils import current_timestamp, get_format_time
 from common.constants import LLMType, ParserType, StatusEnum, TaskStatus, SVR_CONSUMER_GROUP_NAME
 from rag.nlp import rag_tokenizer, search
 from rag.utils.redis_conn import REDIS_CONN
-from rag.utils.doc_store_conn import OrderByExpr
+from common.doc_store.doc_store_base import OrderByExpr
 from common import settings
 
 
@@ -79,7 +80,7 @@ class DocumentService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_list(cls, kb_id, page_number, items_per_page,
-                 orderby, desc, keywords, id, name, suffix=None, run = None):
+                 orderby, desc, keywords, id, name, suffix=None, run = None, doc_ids=None):
         fields = cls.get_cls_model_fields()
         docs = cls.model.select(*[*fields, UserCanvas.title]).join(File2Document, on = (File2Document.document_id == cls.model.id))\
             .join(File, on = (File.id == File2Document.file_id))\
@@ -96,6 +97,8 @@ class DocumentService(CommonService):
             docs = docs.where(
                 fn.LOWER(cls.model.name).contains(keywords.lower())
             )
+        if doc_ids:
+            docs = docs.where(cls.model.id.in_(doc_ids))
         if suffix:
             docs = docs.where(cls.model.suffix.in_(suffix))
         if run:
@@ -122,33 +125,37 @@ class DocumentService(CommonService):
 
     @classmethod
     @DB.connection_context()
-    def get_by_kb_id(cls, kb_id, page_number, items_per_page,
-                     orderby, desc, keywords, run_status, types, suffix):
+    def get_by_kb_id(cls, kb_id, page_number, items_per_page, orderby, desc, keywords, run_status, types, suffix, doc_ids=None, return_empty_metadata=False):
         fields = cls.get_cls_model_fields()
         if keywords:
-            docs = cls.model.select(*[*fields, UserCanvas.title.alias("pipeline_name"), User.nickname])\
-                .join(File2Document, on=(File2Document.document_id == cls.model.id))\
-                .join(File, on=(File.id == File2Document.file_id))\
-                .join(UserCanvas, on=(cls.model.pipeline_id == UserCanvas.id), join_type=JOIN.LEFT_OUTER)\
-                .join(User, on=(cls.model.created_by == User.id), join_type=JOIN.LEFT_OUTER)\
-                .where(
-                    (cls.model.kb_id == kb_id),
-                    (fn.LOWER(cls.model.name).contains(keywords.lower()))
-                )
+            docs = (
+                cls.model.select(*[*fields, UserCanvas.title.alias("pipeline_name"), User.nickname])
+                .join(File2Document, on=(File2Document.document_id == cls.model.id))
+                .join(File, on=(File.id == File2Document.file_id))
+                .join(UserCanvas, on=(cls.model.pipeline_id == UserCanvas.id), join_type=JOIN.LEFT_OUTER)
+                .join(User, on=(cls.model.created_by == User.id), join_type=JOIN.LEFT_OUTER)
+                .where((cls.model.kb_id == kb_id), (fn.LOWER(cls.model.name).contains(keywords.lower())))
+            )
         else:
-            docs = cls.model.select(*[*fields, UserCanvas.title.alias("pipeline_name"), User.nickname])\
-                .join(File2Document, on=(File2Document.document_id == cls.model.id))\
-                .join(UserCanvas, on=(cls.model.pipeline_id == UserCanvas.id), join_type=JOIN.LEFT_OUTER)\
-                .join(File, on=(File.id == File2Document.file_id))\
-                .join(User, on=(cls.model.created_by == User.id), join_type=JOIN.LEFT_OUTER)\
+            docs = (
+                cls.model.select(*[*fields, UserCanvas.title.alias("pipeline_name"), User.nickname])
+                .join(File2Document, on=(File2Document.document_id == cls.model.id))
+                .join(UserCanvas, on=(cls.model.pipeline_id == UserCanvas.id), join_type=JOIN.LEFT_OUTER)
+                .join(File, on=(File.id == File2Document.file_id))
+                .join(User, on=(cls.model.created_by == User.id), join_type=JOIN.LEFT_OUTER)
                 .where(cls.model.kb_id == kb_id)
+            )
 
+        if doc_ids:
+            docs = docs.where(cls.model.id.in_(doc_ids))
         if run_status:
             docs = docs.where(cls.model.run.in_(run_status))
         if types:
             docs = docs.where(cls.model.type.in_(types))
         if suffix:
             docs = docs.where(cls.model.suffix.in_(suffix))
+        if return_empty_metadata:
+            docs = docs.where(fn.COALESCE(fn.JSON_LENGTH(cls.model.meta_fields), 0) == 0)
 
         count = docs.count()
         if desc:
@@ -156,7 +163,6 @@ class DocumentService(CommonService):
         else:
             docs = docs.order_by(cls.model.getter_by(orderby).asc())
 
-
         if page_number and items_per_page:
             docs = docs.paginate(page_number, items_per_page)
 
@@ -176,6 +182,16 @@ class DocumentService(CommonService):
              "1": 2,
              "2": 2
             }
+            "metadata": {
+                "key1": {
+                 "key1_value1": 1,
+                 "key1_value2": 2,
+                },
+                "key2": {
+                 "key2_value1": 2,
+                 "key2_value2": 1,
+                },
+            }
         }, total
         where "1" => RUNNING, "2" => CANCEL
         """
@@ -196,19 +212,42 @@ class DocumentService(CommonService):
         if suffix:
             query = query.where(cls.model.suffix.in_(suffix))
 
-        rows = query.select(cls.model.run, cls.model.suffix)
+        rows = query.select(cls.model.run, cls.model.suffix, cls.model.meta_fields)
         total = rows.count()
 
         suffix_counter = {}
         run_status_counter = {}
+        metadata_counter = {}
+        empty_metadata_count = 0
 
         for row in rows:
             suffix_counter[row.suffix] = suffix_counter.get(row.suffix, 0) + 1
             run_status_counter[str(row.run)] = run_status_counter.get(str(row.run), 0) + 1
+            meta_fields = row.meta_fields or {}
+            if not meta_fields:
+                empty_metadata_count += 1
+                continue
+            has_valid_meta = False
+            for key, value in meta_fields.items():
+                values = value if isinstance(value, list) else [value]
+                for vv in values:
+                    if vv is None:
+                        continue
+                    if isinstance(vv, str) and not vv.strip():
+                        continue
+                    sv = str(vv)
+                    if key not in metadata_counter:
+                        metadata_counter[key] = {}
+                    metadata_counter[key][sv] = metadata_counter[key].get(sv, 0) + 1
+                    has_valid_meta = True
+            if not has_valid_meta:
+                empty_metadata_count += 1
 
+        metadata_counter["empty_metadata"] = {"true": empty_metadata_count}
         return {
             "suffix": suffix_counter,
-            "run_status": run_status_counter
+            "run_status": run_status_counter,
+            "metadata": metadata_counter,
         }, total
 
     @classmethod
@@ -310,7 +349,7 @@ class DocumentService(CommonService):
                 chunks = settings.docStoreConn.search(["img_id"], [], {"doc_id": doc.id}, [], OrderByExpr(),
                                                       page * page_size, page_size, search.index_name(tenant_id),
                                                       [doc.kb_id])
-                chunk_ids = settings.docStoreConn.get_chunk_ids(chunks)
+                chunk_ids = settings.docStoreConn.get_doc_ids(chunks)
                 if not chunk_ids:
                     break
                 all_chunk_ids.extend(chunk_ids)
@@ -644,6 +683,13 @@ class DocumentService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_meta_by_kbs(cls, kb_ids):
+        """
+        Legacy metadata aggregator (backward-compatible).
+        - Does NOT expand list values and a list is kept as one string key.
+          Example: {"tags": ["foo","bar"]} -> meta["tags"]["['foo', 'bar']"] = [doc_id]
+        - Expects meta_fields is a dict.
+        Use when existing callers rely on the old list-as-string semantics.
+        """
         fields = [
             cls.model.id,
             cls.model.meta_fields,
@@ -654,12 +700,184 @@ class DocumentService(CommonService):
             for k,v in r.meta_fields.items():
                 if k not in meta:
                     meta[k] = {}
-                v = str(v)
-                if v not in meta[k]:
-                    meta[k][v] = []
-                meta[k][v].append(doc_id)
+                if not isinstance(v, list):
+                    v = [v]
+                for vv in v:
+                    if vv not in meta[k]:
+                        if isinstance(vv, list) or isinstance(vv, dict):
+                            continue
+                        meta[k][vv] = []
+                    meta[k][vv].append(doc_id)
         return meta
 
+    @classmethod
+    @DB.connection_context()
+    def get_flatted_meta_by_kbs(cls, kb_ids):
+        """
+        - Parses stringified JSON meta_fields when possible and skips non-dict or unparsable values.
+        - Expands list values into individual entries.
+          Example: {"tags": ["foo","bar"], "author": "alice"} ->
+            meta["tags"]["foo"] = [doc_id], meta["tags"]["bar"] = [doc_id], meta["author"]["alice"] = [doc_id]
+        Prefer for metadata_condition filtering and scenarios that must respect list semantics.
+        """
+        fields = [
+            cls.model.id,
+            cls.model.meta_fields,
+        ]
+        meta = {}
+        for r in cls.model.select(*fields).where(cls.model.kb_id.in_(kb_ids)):
+            doc_id = r.id
+            meta_fields = r.meta_fields or {}
+            if isinstance(meta_fields, str):
+                try:
+                    meta_fields = json.loads(meta_fields)
+                except Exception:
+                    continue
+            if not isinstance(meta_fields, dict):
+                continue
+            for k, v in meta_fields.items():
+                if k not in meta:
+                    meta[k] = {}
+                values = v if isinstance(v, list) else [v]
+                for vv in values:
+                    if vv is None:
+                        continue
+                    sv = str(vv)
+                    if sv not in meta[k]:
+                        meta[k][sv] = []
+                    meta[k][sv].append(doc_id)
+        return meta
+
+    @classmethod
+    @DB.connection_context()
+    def get_metadata_summary(cls, kb_id):
+        fields = [cls.model.id, cls.model.meta_fields]
+        summary = {}
+        for r in cls.model.select(*fields).where(cls.model.kb_id == kb_id):
+            meta_fields = r.meta_fields or {}
+            if isinstance(meta_fields, str):
+                try:
+                    meta_fields = json.loads(meta_fields)
+                except Exception:
+                    continue
+            if not isinstance(meta_fields, dict):
+                continue
+            for k, v in meta_fields.items():
+                values = v if isinstance(v, list) else [v]
+                for vv in values:
+                    if not vv:
+                        continue
+                    sv = str(vv)
+                    if k not in summary:
+                        summary[k] = {}
+                    summary[k][sv] = summary[k].get(sv, 0) + 1
+        return {k: sorted([(val, cnt) for val, cnt in v.items()], key=lambda x: x[1], reverse=True) for k, v in summary.items()}
+
+    @classmethod
+    @DB.connection_context()
+    def batch_update_metadata(cls, kb_id, doc_ids, updates=None, deletes=None):
+        updates = updates or []
+        deletes = deletes or []
+        if not doc_ids:
+            return 0
+
+        def _normalize_meta(meta):
+            if isinstance(meta, str):
+                try:
+                    meta = json.loads(meta)
+                except Exception:
+                    return {}
+            if not isinstance(meta, dict):
+                return {}
+            return deepcopy(meta)
+
+        def _str_equal(a, b):
+            return str(a) == str(b)
+
+        def _apply_updates(meta):
+            changed = False
+            for upd in updates:
+                key = upd.get("key")
+                if not key or key not in meta:
+                    continue
+
+                new_value = upd.get("value")
+                match_provided = "match" in upd
+                if isinstance(meta[key], list):
+                    if not match_provided:
+                        if isinstance(new_value, list):
+                            meta[key] = dedupe_list(new_value)
+                        else:
+                            meta[key] = new_value
+                        changed = True
+                    else:
+                        match_value = upd.get("match")
+                        replaced = False
+                        new_list = []
+                        for item in meta[key]:
+                            if _str_equal(item, match_value):
+                                new_list.append(new_value)
+                                replaced = True
+                            else:
+                                new_list.append(item)
+                        if replaced:
+                            meta[key] = dedupe_list(new_list)
+                            changed = True
+                else:
+                    if not match_provided:
+                        meta[key] = new_value
+                        changed = True
+                    else:
+                        match_value = upd.get("match")
+                        if _str_equal(meta[key], match_value):
+                            meta[key] = new_value
+                            changed = True
+            return changed
+
+        def _apply_deletes(meta):
+            changed = False
+            for d in deletes:
+                key = d.get("key")
+                if not key or key not in meta:
+                    continue
+                value = d.get("value", None)
+                if isinstance(meta[key], list):
+                    if value is None:
+                        del meta[key]
+                        changed = True
+                        continue
+                    new_list = [item for item in meta[key] if not _str_equal(item, value)]
+                    if len(new_list) != len(meta[key]):
+                        if new_list:
+                            meta[key] = new_list
+                        else:
+                            del meta[key]
+                        changed = True
+                else:
+                    if value is None or _str_equal(meta[key], value):
+                        del meta[key]
+                        changed = True
+            return changed
+
+        updated_docs = 0
+        with DB.atomic():
+            rows = cls.model.select(cls.model.id, cls.model.meta_fields).where(
+                (cls.model.id.in_(doc_ids)) & (cls.model.kb_id == kb_id)
+            )
+            for r in rows:
+                meta = _normalize_meta(r.meta_fields or {})
+                original_meta = deepcopy(meta)
+                changed = _apply_updates(meta)
+                changed = _apply_deletes(meta) or changed
+                if changed and meta != original_meta:
+                    cls.model.update(
+                        meta_fields=meta,
+                        update_time=current_timestamp(),
+                        update_date=get_format_time()
+                    ).where(cls.model.id == r.id).execute()
+                    updated_docs += 1
+        return updated_docs
+
     @classmethod
     @DB.connection_context()
     def update_progress(cls):
@@ -906,12 +1124,12 @@ def doc_upload_and_parse(conversation_id, file_objs, user_id):
 
     e, dia = DialogService.get_by_id(conv.dialog_id)
     if not dia.kb_ids:
-        raise LookupError("No knowledge base associated with this conversation. "
-                          "Please add a knowledge base before uploading documents")
+        raise LookupError("No dataset associated with this conversation. "
+                          "Please add a dataset before uploading documents")
     kb_id = dia.kb_ids[0]
     e, kb = KnowledgebaseService.get_by_id(kb_id)
     if not e:
-        raise LookupError("Can't find this knowledgebase!")
+        raise LookupError("Can't find this dataset!")
 
     embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING, llm_name=kb.embd_id, lang=kb.language)
 
@@ -999,7 +1217,7 @@ def doc_upload_and_parse(conversation_id, file_objs, user_id):
             from graphrag.general.mind_map_extractor import MindMapExtractor
             mindmap = MindMapExtractor(llm_bdl)
             try:
-                mind_map = trio.run(mindmap, [c["content_with_weight"] for c in docs if c["doc_id"] == doc_id])
+                mind_map = asyncio.run(mindmap([c["content_with_weight"] for c in docs if c["doc_id"] == doc_id]))
                 mind_map = json.dumps(mind_map.output, ensure_ascii=False, indent=2)
                 if len(mind_map) < 32:
                     raise Exception("Few content: " + mind_map)
@@ -1023,8 +1241,8 @@ def doc_upload_and_parse(conversation_id, file_objs, user_id):
             d["q_%d_vec" % len(v)] = v
         for b in range(0, len(cks), es_bulk_size):
             if try_create_idx:
-                if not settings.docStoreConn.indexExist(idxnm, kb_id):
-                    settings.docStoreConn.createIdx(idxnm, kb_id, len(vectors[0]))
+                if not settings.docStoreConn.index_exist(idxnm, kb_id):
+                    settings.docStoreConn.create_idx(idxnm, kb_id, len(vectors[0]))
                 try_create_idx = False
             settings.docStoreConn.insert(cks[b:b + es_bulk_size], idxnm, kb_id)
 

api/db/services/evaluation_service.py

@@ -25,14 +25,17 @@ Provides functionality for evaluating RAG system performance including:
 - Configuration recommendations
 """
 
+import asyncio
 import logging
+import queue
+import threading
 from typing import List, Dict, Any, Optional, Tuple
 from datetime import datetime
 from timeit import default_timer as timer
 
 from api.db.db_models import EvaluationDataset, EvaluationCase, EvaluationRun, EvaluationResult
 from api.db.services.common_service import CommonService
-from api.db.services.dialog_service import DialogService, chat
+from api.db.services.dialog_service import DialogService
 from common.misc_utils import get_uuid
 from common.time_utils import current_timestamp
 from common.constants import StatusEnum
@@ -40,24 +43,24 @@ from common.constants import StatusEnum
 
 class EvaluationService(CommonService):
     """Service for managing RAG evaluations"""
-    
+
     model = EvaluationDataset
-    
+
     # ==================== Dataset Management ====================
-    
+
     @classmethod
-    def create_dataset(cls, name: str, description: str, kb_ids: List[str], 
+    def create_dataset(cls, name: str, description: str, kb_ids: List[str],
                       tenant_id: str, user_id: str) -> Tuple[bool, str]:
         """
         Create a new evaluation dataset.
-        
+
         Args:
             name: Dataset name
             description: Dataset description
             kb_ids: List of knowledge base IDs to evaluate against
             tenant_id: Tenant ID
             user_id: User ID who creates the dataset
-            
+
         Returns:
             (success, dataset_id or error_message)
         """
@@ -74,15 +77,15 @@ class EvaluationService(CommonService):
                 "update_time": current_timestamp(),
                 "status": StatusEnum.VALID.value
             }
-            
+
             if not EvaluationDataset.create(**dataset):
                 return False, "Failed to create dataset"
-            
+
             return True, dataset_id
         except Exception as e:
             logging.error(f"Error creating evaluation dataset: {e}")
             return False, str(e)
-    
+
     @classmethod
     def get_dataset(cls, dataset_id: str) -> Optional[Dict[str, Any]]:
         """Get dataset by ID"""
@@ -94,9 +97,9 @@ class EvaluationService(CommonService):
         except Exception as e:
             logging.error(f"Error getting dataset {dataset_id}: {e}")
             return None
-    
+
     @classmethod
-    def list_datasets(cls, tenant_id: str, user_id: str, 
+    def list_datasets(cls, tenant_id: str, user_id: str,
                      page: int = 1, page_size: int = 20) -> Dict[str, Any]:
         """List datasets for a tenant"""
         try:
@@ -104,10 +107,10 @@ class EvaluationService(CommonService):
                 (EvaluationDataset.tenant_id == tenant_id) &
                 (EvaluationDataset.status == StatusEnum.VALID.value)
             ).order_by(EvaluationDataset.create_time.desc())
-            
+
             total = query.count()
             datasets = query.paginate(page, page_size)
-            
+
             return {
                 "total": total,
                 "datasets": [d.to_dict() for d in datasets]
@@ -115,7 +118,7 @@ class EvaluationService(CommonService):
         except Exception as e:
             logging.error(f"Error listing datasets: {e}")
             return {"total": 0, "datasets": []}
-    
+
     @classmethod
     def update_dataset(cls, dataset_id: str, **kwargs) -> bool:
         """Update dataset"""
@@ -127,7 +130,7 @@ class EvaluationService(CommonService):
         except Exception as e:
             logging.error(f"Error updating dataset {dataset_id}: {e}")
             return False
-    
+
     @classmethod
     def delete_dataset(cls, dataset_id: str) -> bool:
         """Soft delete dataset"""
@@ -139,18 +142,18 @@ class EvaluationService(CommonService):
         except Exception as e:
             logging.error(f"Error deleting dataset {dataset_id}: {e}")
             return False
-    
+
     # ==================== Test Case Management ====================
-    
+
     @classmethod
-    def add_test_case(cls, dataset_id: str, question: str, 
+    def add_test_case(cls, dataset_id: str, question: str,
                      reference_answer: Optional[str] = None,
                      relevant_doc_ids: Optional[List[str]] = None,
                      relevant_chunk_ids: Optional[List[str]] = None,
                      metadata: Optional[Dict[str, Any]] = None) -> Tuple[bool, str]:
         """
         Add a test case to a dataset.
-        
+
         Args:
             dataset_id: Dataset ID
             question: Test question
@@ -158,7 +161,7 @@ class EvaluationService(CommonService):
             relevant_doc_ids: Optional list of relevant document IDs
             relevant_chunk_ids: Optional list of relevant chunk IDs
             metadata: Optional additional metadata
-            
+
         Returns:
             (success, case_id or error_message)
         """
@@ -174,15 +177,15 @@ class EvaluationService(CommonService):
                 "metadata": metadata,
                 "create_time": current_timestamp()
             }
-            
+
             if not EvaluationCase.create(**case):
                 return False, "Failed to create test case"
-            
+
             return True, case_id
         except Exception as e:
             logging.error(f"Error adding test case: {e}")
             return False, str(e)
-    
+
     @classmethod
     def get_test_cases(cls, dataset_id: str) -> List[Dict[str, Any]]:
         """Get all test cases for a dataset"""
@@ -190,12 +193,12 @@ class EvaluationService(CommonService):
             cases = EvaluationCase.select().where(
                 EvaluationCase.dataset_id == dataset_id
             ).order_by(EvaluationCase.create_time)
-            
+
             return [c.to_dict() for c in cases]
         except Exception as e:
             logging.error(f"Error getting test cases for dataset {dataset_id}: {e}")
             return []
-    
+
     @classmethod
     def delete_test_case(cls, case_id: str) -> bool:
         """Delete a test case"""
@@ -206,22 +209,22 @@ class EvaluationService(CommonService):
         except Exception as e:
             logging.error(f"Error deleting test case {case_id}: {e}")
             return False
-    
+
     @classmethod
     def import_test_cases(cls, dataset_id: str, cases: List[Dict[str, Any]]) -> Tuple[int, int]:
         """
         Bulk import test cases from a list.
-        
+
         Args:
             dataset_id: Dataset ID
             cases: List of test case dictionaries
-            
+
         Returns:
             (success_count, failure_count)
         """
         success_count = 0
         failure_count = 0
-        
+
         for case_data in cases:
             success, _ = cls.add_test_case(
                 dataset_id=dataset_id,
@@ -231,28 +234,28 @@ class EvaluationService(CommonService):
                 relevant_chunk_ids=case_data.get("relevant_chunk_ids"),
                 metadata=case_data.get("metadata")
             )
-            
+
             if success:
                 success_count += 1
             else:
                 failure_count += 1
-        
+
         return success_count, failure_count
-    
+
     # ==================== Evaluation Execution ====================
-    
+
     @classmethod
-    def start_evaluation(cls, dataset_id: str, dialog_id: str, 
+    def start_evaluation(cls, dataset_id: str, dialog_id: str,
                         user_id: str, name: Optional[str] = None) -> Tuple[bool, str]:
         """
         Start an evaluation run.
-        
+
         Args:
             dataset_id: Dataset ID
             dialog_id: Dialog configuration to evaluate
             user_id: User ID who starts the run
             name: Optional run name
-            
+
         Returns:
             (success, run_id or error_message)
         """
@@ -261,12 +264,12 @@ class EvaluationService(CommonService):
             success, dialog = DialogService.get_by_id(dialog_id)
             if not success:
                 return False, "Dialog not found"
-            
+
             # Create evaluation run
             run_id = get_uuid()
             if not name:
                 name = f"Evaluation Run {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-            
+
             run = {
                 "id": run_id,
                 "dataset_id": dataset_id,
@@ -279,92 +282,128 @@ class EvaluationService(CommonService):
                 "create_time": current_timestamp(),
                 "complete_time": None
             }
-            
+
             if not EvaluationRun.create(**run):
                 return False, "Failed to create evaluation run"
-            
+
             # Execute evaluation asynchronously (in production, use task queue)
             # For now, we'll execute synchronously
             cls._execute_evaluation(run_id, dataset_id, dialog)
-            
+
             return True, run_id
         except Exception as e:
             logging.error(f"Error starting evaluation: {e}")
             return False, str(e)
-    
+
     @classmethod
     def _execute_evaluation(cls, run_id: str, dataset_id: str, dialog: Any):
         """
         Execute evaluation for all test cases.
-        
+
         This method runs the RAG pipeline for each test case and computes metrics.
         """
         try:
             # Get all test cases
             test_cases = cls.get_test_cases(dataset_id)
-            
+
             if not test_cases:
                 EvaluationRun.update(
                     status="FAILED",
                     complete_time=current_timestamp()
                 ).where(EvaluationRun.id == run_id).execute()
                 return
-            
+
             # Execute each test case
             results = []
             for case in test_cases:
                 result = cls._evaluate_single_case(run_id, case, dialog)
                 if result:
                     results.append(result)
-            
+
             # Compute summary metrics
             metrics_summary = cls._compute_summary_metrics(results)
-            
+
             # Update run status
             EvaluationRun.update(
                 status="COMPLETED",
                 metrics_summary=metrics_summary,
                 complete_time=current_timestamp()
             ).where(EvaluationRun.id == run_id).execute()
-            
+
         except Exception as e:
             logging.error(f"Error executing evaluation {run_id}: {e}")
             EvaluationRun.update(
                 status="FAILED",
                 complete_time=current_timestamp()
             ).where(EvaluationRun.id == run_id).execute()
-    
+
     @classmethod
-    def _evaluate_single_case(cls, run_id: str, case: Dict[str, Any], 
+    def _evaluate_single_case(cls, run_id: str, case: Dict[str, Any],
                              dialog: Any) -> Optional[Dict[str, Any]]:
         """
         Evaluate a single test case.
-        
+
         Args:
             run_id: Evaluation run ID
             case: Test case dictionary
             dialog: Dialog configuration
-            
+
         Returns:
             Result dictionary or None if failed
         """
         try:
             # Prepare messages
             messages = [{"role": "user", "content": case["question"]}]
-            
+
             # Execute RAG pipeline
             start_time = timer()
             answer = ""
             retrieved_chunks = []
-            
+
+
+            def _sync_from_async_gen(async_gen):
+                result_queue: queue.Queue = queue.Queue()
+
+                def runner():
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+
+                    async def consume():
+                        try:
+                            async for item in async_gen:
+                                result_queue.put(item)
+                        except Exception as e:
+                            result_queue.put(e)
+                        finally:
+                            result_queue.put(StopIteration)
+
+                    loop.run_until_complete(consume())
+                    loop.close()
+
+                threading.Thread(target=runner, daemon=True).start()
+
+                while True:
+                    item = result_queue.get()
+                    if item is StopIteration:
+                        break
+                    if isinstance(item, Exception):
+                        raise item
+                    yield item
+
+
+            def chat(dialog, messages, stream=True, **kwargs):
+                from api.db.services.dialog_service import async_chat
+
+                return _sync_from_async_gen(async_chat(dialog, messages, stream=stream, **kwargs))
+
             for ans in chat(dialog, messages, stream=False):
                 if isinstance(ans, dict):
                     answer = ans.get("answer", "")
                     retrieved_chunks = ans.get("reference", {}).get("chunks", [])
                     break
-            
+
             execution_time = timer() - start_time
-            
+
             # Compute metrics
             metrics = cls._compute_metrics(
                 question=case["question"],
@@ -374,7 +413,7 @@ class EvaluationService(CommonService):
                 relevant_chunk_ids=case.get("relevant_chunk_ids"),
                 dialog=dialog
             )
-            
+
             # Save result
             result_id = get_uuid()
             result = {
@@ -388,14 +427,14 @@ class EvaluationService(CommonService):
                 "token_usage": None,  # TODO: Track token usage
                 "create_time": current_timestamp()
             }
-            
+
             EvaluationResult.create(**result)
-            
+
             return result
         except Exception as e:
             logging.error(f"Error evaluating case {case.get('id')}: {e}")
             return None
-    
+
     @classmethod
     def _compute_metrics(cls, question: str, generated_answer: str,
                         reference_answer: Optional[str],
@@ -404,69 +443,69 @@ class EvaluationService(CommonService):
                         dialog: Any) -> Dict[str, float]:
         """
         Compute evaluation metrics for a single test case.
-        
+
         Returns:
             Dictionary of metric names to values
         """
         metrics = {}
-        
+
         # Retrieval metrics (if ground truth chunks provided)
         if relevant_chunk_ids:
             retrieved_ids = [c.get("chunk_id") for c in retrieved_chunks]
             metrics.update(cls._compute_retrieval_metrics(retrieved_ids, relevant_chunk_ids))
-        
+
         # Generation metrics
         if generated_answer:
             # Basic metrics
             metrics["answer_length"] = len(generated_answer)
             metrics["has_answer"] = 1.0 if generated_answer.strip() else 0.0
-            
+
             # TODO: Implement advanced metrics using LLM-as-judge
             # - Faithfulness (hallucination detection)
             # - Answer relevance
             # - Context relevance
             # - Semantic similarity (if reference answer provided)
-        
+
         return metrics
-    
+
     @classmethod
-    def _compute_retrieval_metrics(cls, retrieved_ids: List[str], 
+    def _compute_retrieval_metrics(cls, retrieved_ids: List[str],
                                    relevant_ids: List[str]) -> Dict[str, float]:
         """
         Compute retrieval metrics.
-        
+
         Args:
             retrieved_ids: List of retrieved chunk IDs
             relevant_ids: List of relevant chunk IDs (ground truth)
-            
+
         Returns:
             Dictionary of retrieval metrics
         """
         if not relevant_ids:
             return {}
-        
+
         retrieved_set = set(retrieved_ids)
         relevant_set = set(relevant_ids)
-        
+
         # Precision: proportion of retrieved that are relevant
         precision = len(retrieved_set & relevant_set) / len(retrieved_set) if retrieved_set else 0.0
-        
+
         # Recall: proportion of relevant that were retrieved
         recall = len(retrieved_set & relevant_set) / len(relevant_set) if relevant_set else 0.0
-        
+
         # F1 score
         f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
-        
+
         # Hit rate: whether any relevant chunk was retrieved
         hit_rate = 1.0 if (retrieved_set & relevant_set) else 0.0
-        
+
         # MRR (Mean Reciprocal Rank): position of first relevant chunk
         mrr = 0.0
         for i, chunk_id in enumerate(retrieved_ids, 1):
             if chunk_id in relevant_set:
                 mrr = 1.0 / i
                 break
-        
+
         return {
             "precision": precision,
             "recall": recall,
@@ -474,45 +513,45 @@ class EvaluationService(CommonService):
             "hit_rate": hit_rate,
             "mrr": mrr
         }
-    
+
     @classmethod
     def _compute_summary_metrics(cls, results: List[Dict[str, Any]]) -> Dict[str, Any]:
         """
         Compute summary metrics across all test cases.
-        
+
         Args:
             results: List of result dictionaries
-            
+
         Returns:
             Summary metrics dictionary
         """
         if not results:
             return {}
-        
+
         # Aggregate metrics
         metric_sums = {}
         metric_counts = {}
-        
+
         for result in results:
             metrics = result.get("metrics", {})
             for key, value in metrics.items():
                 if isinstance(value, (int, float)):
                     metric_sums[key] = metric_sums.get(key, 0) + value
                     metric_counts[key] = metric_counts.get(key, 0) + 1
-        
+
         # Compute averages
         summary = {
             "total_cases": len(results),
             "avg_execution_time": sum(r.get("execution_time", 0) for r in results) / len(results)
         }
-        
+
         for key in metric_sums:
             summary[f"avg_{key}"] = metric_sums[key] / metric_counts[key]
-        
+
         return summary
-    
+
     # ==================== Results & Analysis ====================
-    
+
     @classmethod
     def get_run_results(cls, run_id: str) -> Dict[str, Any]:
         """Get results for an evaluation run"""
@@ -520,11 +559,11 @@ class EvaluationService(CommonService):
             run = EvaluationRun.get_by_id(run_id)
             if not run:
                 return {}
-            
+
             results = EvaluationResult.select().where(
                 EvaluationResult.run_id == run_id
             ).order_by(EvaluationResult.create_time)
-            
+
             return {
                 "run": run.to_dict(),
                 "results": [r.to_dict() for r in results]
@@ -532,15 +571,15 @@ class EvaluationService(CommonService):
         except Exception as e:
             logging.error(f"Error getting run results {run_id}: {e}")
             return {}
-    
+
     @classmethod
     def get_recommendations(cls, run_id: str) -> List[Dict[str, Any]]:
         """
         Analyze evaluation results and provide configuration recommendations.
-        
+
         Args:
             run_id: Evaluation run ID
-            
+
         Returns:
             List of recommendation dictionaries
         """
@@ -548,10 +587,10 @@ class EvaluationService(CommonService):
             run = EvaluationRun.get_by_id(run_id)
             if not run or not run.metrics_summary:
                 return []
-            
+
             metrics = run.metrics_summary
             recommendations = []
-            
+
             # Low precision: retrieving irrelevant chunks
             if metrics.get("avg_precision", 1.0) < 0.7:
                 recommendations.append({
@@ -564,7 +603,7 @@ class EvaluationService(CommonService):
                         "Reduce top_k to return fewer chunks"
                     ]
                 })
-            
+
             # Low recall: missing relevant chunks
             if metrics.get("avg_recall", 1.0) < 0.7:
                 recommendations.append({
@@ -578,7 +617,7 @@ class EvaluationService(CommonService):
                         "Check chunk size - may be too large or too small"
                     ]
                 })
-            
+
             # Slow response time
             if metrics.get("avg_execution_time", 0) > 5.0:
                 recommendations.append({
@@ -591,7 +630,7 @@ class EvaluationService(CommonService):
                         "Consider caching frequently asked questions"
                     ]
                 })
-            
+
             return recommendations
         except Exception as e:
             logging.error(f"Error generating recommendations for run {run_id}: {e}")

api/db/services/file_service.py

@@ -94,13 +94,13 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_kb_id_by_file_id(cls, file_id):
-        # Get knowledge base IDs associated with a file
+        # Get dataset IDs associated with a file
         # Args:
         #     file_id: File ID
         # Returns:
-        #     List of dictionaries containing knowledge base IDs and names
+        #     List of dictionaries containing dataset IDs and names
         kbs = (
-            cls.model.select(*[Knowledgebase.id, Knowledgebase.name])
+            cls.model.select(*[Knowledgebase.id, Knowledgebase.name, File2Document.document_id])
             .join(File2Document, on=(File2Document.file_id == file_id))
             .join(Document, on=(File2Document.document_id == Document.id))
             .join(Knowledgebase, on=(Knowledgebase.id == Document.kb_id))
@@ -110,7 +110,7 @@ class FileService(CommonService):
             return []
         kbs_info_list = []
         for kb in list(kbs.dicts()):
-            kbs_info_list.append({"kb_id": kb["id"], "kb_name": kb["name"]})
+            kbs_info_list.append({"kb_id": kb["id"], "kb_name": kb["name"], "document_id": kb["document_id"]})
         return kbs_info_list
 
     @classmethod
@@ -247,7 +247,7 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_kb_folder(cls, tenant_id):
-        # Get knowledge base folder for tenant
+        # Get dataset folder for tenant
         # Args:
         #     tenant_id: Tenant ID
         # Returns:
@@ -263,7 +263,7 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def new_a_file_from_kb(cls, tenant_id, name, parent_id, ty=FileType.FOLDER.value, size=0, location=""):
-        # Create a new file from knowledge base
+        # Create a new file from dataset
         # Args:
         #     tenant_id: Tenant ID
         #     name: File name
@@ -292,7 +292,7 @@ class FileService(CommonService):
     @classmethod
     @DB.connection_context()
     def init_knowledgebase_docs(cls, root_id, tenant_id):
-        # Initialize knowledge base documents
+        # Initialize dataset documents
         # Args:
         #     root_id: Root folder ID
         #     tenant_id: Tenant ID

api/db/services/knowledgebase_service.py

@@ -30,9 +30,9 @@ from api.utils.api_utils import get_parser_config, get_data_error_result
 
 
 class KnowledgebaseService(CommonService):
-    """Service class for managing knowledge base operations.
+    """Service class for managing dataset operations.
 
-    This class extends CommonService to provide specialized functionality for knowledge base
+    This class extends CommonService to provide specialized functionality for dataset
     management, including document parsing status tracking, access control, and configuration
     management. It handles operations such as listing, creating, updating, and deleting
     knowledge bases, as well as managing their associated documents and permissions.
@@ -41,7 +41,7 @@ class KnowledgebaseService(CommonService):
     - Document parsing status verification
     - Knowledge base access control
     - Parser configuration management
-    - Tenant-based knowledge base organization
+    - Tenant-based dataset organization
 
     Attributes:
         model: The Knowledgebase model class for database operations.
@@ -51,18 +51,18 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def accessible4deletion(cls, kb_id, user_id):
-        """Check if a knowledge base can be deleted by a specific user.
+        """Check if a dataset can be deleted by a specific user.
 
-        This method verifies whether a user has permission to delete a knowledge base
-        by checking if they are the creator of that knowledge base.
+        This method verifies whether a user has permission to delete a dataset
+        by checking if they are the creator of that dataset.
 
         Args:
-            kb_id (str): The unique identifier of the knowledge base to check.
+            kb_id (str): The unique identifier of the dataset to check.
             user_id (str): The unique identifier of the user attempting the deletion.
 
         Returns:
-            bool: True if the user has permission to delete the knowledge base,
-                  False if the user doesn't have permission or the knowledge base doesn't exist.
+            bool: True if the user has permission to delete the dataset,
+                  False if the user doesn't have permission or the dataset doesn't exist.
 
         Example:
             >>> KnowledgebaseService.accessible4deletion("kb123", "user456")
@@ -71,10 +71,10 @@ class KnowledgebaseService(CommonService):
         Note:
             - This method only checks creator permissions
             - A return value of False can mean either:
-                1. The knowledge base doesn't exist
-                2. The user is not the creator of the knowledge base
+                1. The dataset doesn't exist
+                2. The user is not the creator of the dataset
         """
-        # Check if a knowledge base can be deleted by a user
+        # Check if a dataset can be deleted by a user
         docs = cls.model.select(
             cls.model.id).where(cls.model.id == kb_id, cls.model.created_by == user_id).paginate(0, 1)
         docs = docs.dicts()
@@ -85,7 +85,7 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def is_parsed_done(cls, kb_id):
-        # Check if all documents in the knowledge base have completed parsing
+        # Check if all documents in the dataset have completed parsing
         #
         # Args:
         #     kb_id: Knowledge base ID
@@ -96,13 +96,13 @@ class KnowledgebaseService(CommonService):
         from common.constants import TaskStatus
         from api.db.services.document_service import DocumentService
 
-        # Get knowledge base information
+        # Get dataset information
         kbs = cls.query(id=kb_id)
         if not kbs:
             return False, "Knowledge base not found"
         kb = kbs[0]
 
-        # Get all documents in the knowledge base
+        # Get all documents in the dataset
         docs, _ = DocumentService.get_by_kb_id(kb_id, 1, 1000, "create_time", True, "", [], [])
 
         # Check parsing status of each document
@@ -119,9 +119,9 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def list_documents_by_ids(cls, kb_ids):
-        # Get document IDs associated with given knowledge base IDs
+        # Get document IDs associated with given dataset IDs
         # Args:
-        #     kb_ids: List of knowledge base IDs
+        #     kb_ids: List of dataset IDs
         # Returns:
         #     List of document IDs
         doc_ids = cls.model.select(Document.id.alias("document_id")).join(Document, on=(cls.model.id == Document.kb_id)).where(
@@ -235,11 +235,11 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_kb_ids(cls, tenant_id):
-        # Get all knowledge base IDs for a tenant
+        # Get all dataset IDs for a tenant
         # Args:
         #     tenant_id: Tenant ID
         # Returns:
-        #     List of knowledge base IDs
+        #     List of dataset IDs
         fields = [
             cls.model.id,
         ]
@@ -250,11 +250,11 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_detail(cls, kb_id):
-        # Get detailed information about a knowledge base
+        # Get detailed information about a dataset
         # Args:
         #     kb_id: Knowledge base ID
         # Returns:
-        #     Dictionary containing knowledge base details
+        #     Dictionary containing dataset details
         fields = [
             cls.model.id,
             cls.model.embd_id,
@@ -294,13 +294,13 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def update_parser_config(cls, id, config):
-        # Update parser configuration for a knowledge base
+        # Update parser configuration for a dataset
         # Args:
         #     id: Knowledge base ID
         #     config: New parser configuration
         e, m = cls.get_by_id(id)
         if not e:
-            raise LookupError(f"knowledgebase({id}) not found.")
+            raise LookupError(f"dataset({id}) not found.")
 
         def dfs_update(old, new):
             # Deep update of nested configuration
@@ -325,7 +325,7 @@ class KnowledgebaseService(CommonService):
     def delete_field_map(cls, id):
         e, m = cls.get_by_id(id)
         if not e:
-            raise LookupError(f"knowledgebase({id}) not found.")
+            raise LookupError(f"dataset({id}) not found.")
 
         m.parser_config.pop("field_map", None)
         cls.update_by_id(id, {"parser_config": m.parser_config})
@@ -335,7 +335,7 @@ class KnowledgebaseService(CommonService):
     def get_field_map(cls, ids):
         # Get field mappings for knowledge bases
         # Args:
-        #     ids: List of knowledge base IDs
+        #     ids: List of dataset IDs
         # Returns:
         #     Dictionary of field mappings
         conf = {}
@@ -347,7 +347,7 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_by_name(cls, kb_name, tenant_id):
-        # Get knowledge base by name and tenant ID
+        # Get dataset by name and tenant ID
         # Args:
         #     kb_name: Knowledge base name
         #     tenant_id: Tenant ID
@@ -365,9 +365,9 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_all_ids(cls):
-        # Get all knowledge base IDs
+        # Get all dataset IDs
         # Returns:
-        #     List of all knowledge base IDs
+        #     List of all dataset IDs
         return [m["id"] for m in cls.model.select(cls.model.id).dicts()]
 
 
@@ -425,6 +425,7 @@ class KnowledgebaseService(CommonService):
 
         # Update parser_config (always override with validated default/merged config)
         payload["parser_config"] = get_parser_config(parser_id, kwargs.get("parser_config"))
+        payload["parser_config"]["llm_id"] = _t.llm_id
 
         return True, payload
 
@@ -471,7 +472,7 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def accessible(cls, kb_id, user_id):
-        # Check if a knowledge base is accessible by a user
+        # Check if a dataset is accessible by a user
         # Args:
         #     kb_id: Knowledge base ID
         #     user_id: User ID
@@ -488,12 +489,12 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_kb_by_id(cls, kb_id, user_id):
-        # Get knowledge base by ID and user ID
+        # Get dataset by ID and user ID
         # Args:
         #     kb_id: Knowledge base ID
         #     user_id: User ID
         # Returns:
-        #     List containing knowledge base information
+        #     List containing dataset information
         kbs = cls.model.select().join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
                                       ).where(cls.model.id == kb_id, UserTenant.user_id == user_id).paginate(0, 1)
         kbs = kbs.dicts()
@@ -502,12 +503,12 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_kb_by_name(cls, kb_name, user_id):
-        # Get knowledge base by name and user ID
+        # Get dataset by name and user ID
         # Args:
         #     kb_name: Knowledge base name
         #     user_id: User ID
         # Returns:
-        #     List containing knowledge base information
+        #     List containing dataset information
         kbs = cls.model.select().join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
                                       ).where(cls.model.name == kb_name, UserTenant.user_id == user_id).paginate(0, 1)
         kbs = kbs.dicts()

api/db/services/llm_service.py

@@ -16,15 +16,17 @@
 import asyncio
 import inspect
 import logging
+import queue
 import re
 import threading
-from common.token_utils import num_tokens_from_string
 from functools import partial
 from typing import Generator
-from common.constants import LLMType
+
 from api.db.db_models import LLM
 from api.db.services.common_service import CommonService
 from api.db.services.tenant_llm_service import LLM4Tenant, TenantLLMService
+from common.constants import LLMType
+from common.token_utils import num_tokens_from_string
 
 
 class LLMService(CommonService):
@@ -33,6 +35,7 @@ class LLMService(CommonService):
 
 def get_init_tenant_llm(user_id):
     from common import settings
+
     tenant_llm = []
 
     model_configs = {
@@ -106,7 +109,7 @@ class LLMBundle(LLM4Tenant):
 
         llm_name = getattr(self, "llm_name", None)
         if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens, llm_name):
-            logging.error("LLMBundle.encode can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
+            logging.error("LLMBundle.encode can't update token usage for <tenant redacted>/EMBEDDING used_tokens: {}".format(used_tokens))
 
         if self.langfuse:
             generation.update(usage_details={"total_tokens": used_tokens})
@@ -121,7 +124,7 @@ class LLMBundle(LLM4Tenant):
         emd, used_tokens = self.mdl.encode_queries(query)
         llm_name = getattr(self, "llm_name", None)
         if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens, llm_name):
-            logging.error("LLMBundle.encode_queries can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
+            logging.error("LLMBundle.encode_queries can't update token usage for <tenant redacted>/EMBEDDING used_tokens: {}".format(used_tokens))
 
         if self.langfuse:
             generation.update(usage_details={"total_tokens": used_tokens})
@@ -193,7 +196,7 @@ class LLMBundle(LLM4Tenant):
                 generation = self.langfuse.start_generation(
                     trace_context=self.trace_context,
                     name="stream_transcription",
-                    metadata={"model": self.llm_name}
+                    metadata={"model": self.llm_name},
                 )
             final_text = ""
             used_tokens = 0
@@ -217,32 +220,34 @@ class LLMBundle(LLM4Tenant):
                 if self.langfuse:
                     generation.update(
                         output={"output": final_text},
-                        usage_details={"total_tokens": used_tokens}
+                        usage_details={"total_tokens": used_tokens},
                     )
                     generation.end()
 
             return
 
         if self.langfuse:
-            generation = self.langfuse.start_generation(trace_context=self.trace_context, name="stream_transcription", metadata={"model": self.llm_name})
-        full_text, used_tokens = mdl.transcription(audio)
-        if not TenantLLMService.increase_usage(
-            self.tenant_id, self.llm_type, used_tokens
-        ):
-            logging.error(
-                f"LLMBundle.stream_transcription can't update token usage for {self.tenant_id}/SEQUENCE2TXT used_tokens: {used_tokens}"
+            generation = self.langfuse.start_generation(
+                trace_context=self.trace_context,
+                name="stream_transcription",
+                metadata={"model": self.llm_name},
             )
+
+        full_text, used_tokens = mdl.transcription(audio)
+        if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens):
+            logging.error(f"LLMBundle.stream_transcription can't update token usage for {self.tenant_id}/SEQUENCE2TXT used_tokens: {used_tokens}")
+
         if self.langfuse:
             generation.update(
                 output={"output": full_text},
-                usage_details={"total_tokens": used_tokens}
+                usage_details={"total_tokens": used_tokens},
             )
             generation.end()
 
         yield {
             "event": "final",
             "text": full_text,
-            "streaming": False
+            "streaming": False,
         }
 
     def tts(self, text: str) -> Generator[bytes, None, None]:
@@ -289,60 +294,58 @@ class LLMBundle(LLM4Tenant):
             return kwargs
         else:
             return {k: v for k, v in kwargs.items() if k in allowed_params}
-    def chat(self, system: str, history: list, gen_conf: dict = {}, **kwargs) -> str:
-        if self.langfuse:
-            generation = self.langfuse.start_generation(trace_context=self.trace_context, name="chat", model=self.llm_name, input={"system": system, "history": history})
 
-        chat_partial = partial(self.mdl.chat, system, history, gen_conf, **kwargs)
-        if self.is_tools and self.mdl.is_tools:
-            chat_partial = partial(self.mdl.chat_with_tools, system, history, gen_conf, **kwargs)
+    def _run_coroutine_sync(self, coro):
+        try:
+            asyncio.get_running_loop()
+        except RuntimeError:
+            return asyncio.run(coro)
 
-        use_kwargs = self._clean_param(chat_partial, **kwargs)
-        txt, used_tokens = chat_partial(**use_kwargs)
-        txt = self._remove_reasoning_content(txt)
+        result_queue: queue.Queue = queue.Queue()
 
-        if not self.verbose_tool_use:
-            txt = re.sub(r"<tool_call>.*?</tool_call>", "", txt, flags=re.DOTALL)
+        def runner():
+            try:
+                result_queue.put((True, asyncio.run(coro)))
+            except Exception as e:
+                result_queue.put((False, e))
 
-        if used_tokens and not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens, self.llm_name):
-            logging.error("LLMBundle.chat can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name, used_tokens))
+        thread = threading.Thread(target=runner, daemon=True)
+        thread.start()
+        thread.join()
 
-        if self.langfuse:
-            generation.update(output={"output": txt}, usage_details={"total_tokens": used_tokens})
-            generation.end()
+        success, value = result_queue.get_nowait()
+        if success:
+            return value
+        raise value
 
-        return txt
+    def _sync_from_async_stream(self, async_gen_fn, *args, **kwargs):
+        result_queue: queue.Queue = queue.Queue()
 
-    def chat_streamly(self, system: str, history: list, gen_conf: dict = {}, **kwargs):
-        if self.langfuse:
-            generation = self.langfuse.start_generation(trace_context=self.trace_context, name="chat_streamly", model=self.llm_name, input={"system": system, "history": history})
+        def runner():
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
 
-        ans = ""
-        chat_partial = partial(self.mdl.chat_streamly, system, history, gen_conf)
-        total_tokens = 0
-        if self.is_tools and self.mdl.is_tools:
-            chat_partial = partial(self.mdl.chat_streamly_with_tools, system, history, gen_conf)
-        use_kwargs = self._clean_param(chat_partial, **kwargs)
-        for txt in chat_partial(**use_kwargs):
-            if isinstance(txt, int):
-                total_tokens = txt
-                if self.langfuse:
-                    generation.update(output={"output": ans})
-                    generation.end()
+            async def consume():
+                try:
+                    async for item in async_gen_fn(*args, **kwargs):
+                        result_queue.put(item)
+                except Exception as e:
+                    result_queue.put(e)
+                finally:
+                    result_queue.put(StopIteration)
+
+            loop.run_until_complete(consume())
+            loop.close()
+
+        threading.Thread(target=runner, daemon=True).start()
+
+        while True:
+            item = result_queue.get()
+            if item is StopIteration:
                 break
-
-            if txt.endswith("</think>"):
-                ans = ans[: -len("</think>")]
-
-            if not self.verbose_tool_use:
-                txt = re.sub(r"<tool_call>.*?</tool_call>", "", txt, flags=re.DOTALL)
-
-            ans += txt
-            yield ans
-
-        if total_tokens > 0:
-            if not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, total_tokens, self.llm_name):
-                logging.error("LLMBundle.chat_streamly can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name, total_tokens))
+            if isinstance(item, Exception):
+                raise item
+            yield item
 
     def _bridge_sync_stream(self, gen):
         loop = asyncio.get_running_loop()
@@ -352,7 +355,7 @@ class LLMBundle(LLM4Tenant):
             try:
                 for item in gen:
                     loop.call_soon_threadsafe(queue.put_nowait, item)
-            except Exception as e:  # pragma: no cover
+            except Exception as e:
                 loop.call_soon_threadsafe(queue.put_nowait, e)
             finally:
                 loop.call_soon_threadsafe(queue.put_nowait, StopAsyncIteration)
@@ -361,18 +364,27 @@ class LLMBundle(LLM4Tenant):
         return queue
 
     async def async_chat(self, system: str, history: list, gen_conf: dict = {}, **kwargs):
-        chat_partial = partial(self.mdl.chat, system, history, gen_conf, **kwargs)
-        if self.is_tools and self.mdl.is_tools and hasattr(self.mdl, "chat_with_tools"):
-            chat_partial = partial(self.mdl.chat_with_tools, system, history, gen_conf, **kwargs)
+        if self.is_tools and getattr(self.mdl, "is_tools", False) and hasattr(self.mdl, "async_chat_with_tools"):
+            base_fn = self.mdl.async_chat_with_tools
+        elif hasattr(self.mdl, "async_chat"):
+            base_fn = self.mdl.async_chat
+        else:
+            raise RuntimeError(f"Model {self.mdl} does not implement async_chat or async_chat_with_tools")
 
+        generation = None
+        if self.langfuse:
+            generation = self.langfuse.start_generation(trace_context=self.trace_context, name="chat", model=self.llm_name, input={"system": system, "history": history})
+
+        chat_partial = partial(base_fn, system, history, gen_conf)
         use_kwargs = self._clean_param(chat_partial, **kwargs)
 
-        if hasattr(self.mdl, "async_chat_with_tools") and self.is_tools and self.mdl.is_tools:
-            txt, used_tokens = await self.mdl.async_chat_with_tools(system, history, gen_conf, **use_kwargs)
-        elif hasattr(self.mdl, "async_chat"):
-            txt, used_tokens = await self.mdl.async_chat(system, history, gen_conf, **use_kwargs)
-        else:
-            txt, used_tokens = await asyncio.to_thread(chat_partial, **use_kwargs)
+        try:
+            txt, used_tokens = await chat_partial(**use_kwargs)
+        except Exception as e:
+            if generation:
+                generation.update(output={"error": str(e)})
+                generation.end()
+            raise
 
         txt = self._remove_reasoning_content(txt)
         if not self.verbose_tool_use:
@@ -381,49 +393,51 @@ class LLMBundle(LLM4Tenant):
         if used_tokens and not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, used_tokens, self.llm_name):
             logging.error("LLMBundle.async_chat can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name, used_tokens))
 
+        if generation:
+            generation.update(output={"output": txt}, usage_details={"total_tokens": used_tokens})
+            generation.end()
+
         return txt
 
     async def async_chat_streamly(self, system: str, history: list, gen_conf: dict = {}, **kwargs):
         total_tokens = 0
         ans = ""
-        if self.is_tools and self.mdl.is_tools:
+        if self.is_tools and getattr(self.mdl, "is_tools", False) and hasattr(self.mdl, "async_chat_streamly_with_tools"):
             stream_fn = getattr(self.mdl, "async_chat_streamly_with_tools", None)
-        else:
+        elif hasattr(self.mdl, "async_chat_streamly"):
             stream_fn = getattr(self.mdl, "async_chat_streamly", None)
+        else:
+            raise RuntimeError(f"Model {self.mdl} does not implement async_chat or async_chat_with_tools")
+
+        generation = None
+        if self.langfuse:
+            generation = self.langfuse.start_generation(trace_context=self.trace_context, name="chat_streamly", model=self.llm_name, input={"system": system, "history": history})
 
         if stream_fn:
             chat_partial = partial(stream_fn, system, history, gen_conf)
             use_kwargs = self._clean_param(chat_partial, **kwargs)
-            async for txt in chat_partial(**use_kwargs):
-                if isinstance(txt, int):
-                    total_tokens = txt
-                    break
+            try:
+                async for txt in chat_partial(**use_kwargs):
+                    if isinstance(txt, int):
+                        total_tokens = txt
+                        break
 
-                if txt.endswith("</think>"):
-                    ans = ans[: -len("</think>")]
+                    if txt.endswith("</think>"):
+                        ans = ans[: -len("</think>")]
 
-                if not self.verbose_tool_use:
-                    txt = re.sub(r"<tool_call>.*?</tool_call>", "", txt, flags=re.DOTALL)
+                    if not self.verbose_tool_use:
+                        txt = re.sub(r"<tool_call>.*?</tool_call>", "", txt, flags=re.DOTALL)
 
-                ans += txt
-                yield ans
+                    ans += txt
+                    yield ans
+            except Exception as e:
+                if generation:
+                    generation.update(output={"error": str(e)})
+                    generation.end()
+                raise
             if total_tokens and not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, total_tokens, self.llm_name):
                 logging.error("LLMBundle.async_chat_streamly can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name, total_tokens))
+            if generation:
+                generation.update(output={"output": ans}, usage_details={"total_tokens": total_tokens})
+                generation.end()
             return
-
-        chat_partial = partial(self.mdl.chat_streamly_with_tools if (self.is_tools and self.mdl.is_tools) else self.mdl.chat_streamly, system, history, gen_conf)
-        use_kwargs = self._clean_param(chat_partial, **kwargs)
-        queue = self._bridge_sync_stream(chat_partial(**use_kwargs))
-        while True:
-            item = await queue.get()
-            if item is StopAsyncIteration:
-                break
-            if isinstance(item, Exception):
-                raise item
-            if isinstance(item, int):
-                total_tokens = item
-                break
-            yield item
-
-        if total_tokens and not TenantLLMService.increase_usage(self.tenant_id, self.llm_type, total_tokens, self.llm_name):
-            logging.error("LLMBundle.async_chat_streamly can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name, total_tokens))

api/db/services/memory_service.py

@@ -0,0 +1,170 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from typing import List
+
+from api.db.db_models import DB, Memory, User
+from api.db.services import duplicate_name
+from api.db.services.common_service import CommonService
+from api.utils.memory_utils import calculate_memory_type
+from api.constants import MEMORY_NAME_LIMIT
+from common.misc_utils import get_uuid
+from common.time_utils import get_format_time, current_timestamp
+from memory.utils.prompt_util import PromptAssembler
+
+
+class MemoryService(CommonService):
+    # Service class for manage memory operations
+    model = Memory
+
+    @classmethod
+    @DB.connection_context()
+    def get_by_memory_id(cls, memory_id: str):
+        return cls.model.select().where(cls.model.id == memory_id).first()
+
+    @classmethod
+    @DB.connection_context()
+    def get_by_tenant_id(cls, tenant_id: str):
+        return cls.model.select().where(cls.model.tenant_id == tenant_id)
+
+    @classmethod
+    @DB.connection_context()
+    def get_all_memory(cls):
+        memory_list = cls.model.select()
+        return list(memory_list)
+
+    @classmethod
+    @DB.connection_context()
+    def get_with_owner_name_by_id(cls, memory_id: str):
+        fields = [
+            cls.model.id,
+            cls.model.name,
+            cls.model.avatar,
+            cls.model.tenant_id,
+            User.nickname.alias("owner_name"),
+            cls.model.memory_type,
+            cls.model.storage_type,
+            cls.model.embd_id,
+            cls.model.llm_id,
+            cls.model.permissions,
+            cls.model.description,
+            cls.model.memory_size,
+            cls.model.forgetting_policy,
+            cls.model.temperature,
+            cls.model.system_prompt,
+            cls.model.user_prompt,
+            cls.model.create_date,
+            cls.model.create_time
+        ]
+        memory = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
+            cls.model.id == memory_id
+        ).first()
+        return memory
+
+    @classmethod
+    @DB.connection_context()
+    def get_by_filter(cls, filter_dict: dict, keywords: str, page: int = 1, page_size: int = 50):
+        fields = [
+            cls.model.id,
+            cls.model.name,
+            cls.model.avatar,
+            cls.model.tenant_id,
+            User.nickname.alias("owner_name"),
+            cls.model.memory_type,
+            cls.model.storage_type,
+            cls.model.permissions,
+            cls.model.description,
+            cls.model.create_time,
+            cls.model.create_date
+        ]
+        memories = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id))
+        if filter_dict.get("tenant_id"):
+            memories = memories.where(cls.model.tenant_id.in_(filter_dict["tenant_id"]))
+        if filter_dict.get("memory_type"):
+            memory_type_int = calculate_memory_type(filter_dict["memory_type"])
+            memories = memories.where(cls.model.memory_type.bin_and(memory_type_int) > 0)
+        if filter_dict.get("storage_type"):
+            memories = memories.where(cls.model.storage_type == filter_dict["storage_type"])
+        if keywords:
+            memories = memories.where(cls.model.name.contains(keywords))
+        count = memories.count()
+        memories = memories.order_by(cls.model.update_time.desc())
+        memories = memories.paginate(page, page_size)
+
+        return list(memories.dicts()), count
+
+    @classmethod
+    @DB.connection_context()
+    def create_memory(cls, tenant_id: str, name: str, memory_type: List[str], embd_id: str, llm_id: str):
+        # Deduplicate name within tenant
+        memory_name = duplicate_name(
+            cls.query,
+            name=name,
+            tenant_id=tenant_id
+        )
+        if len(memory_name) > MEMORY_NAME_LIMIT:
+            return False, f"Memory name {memory_name} exceeds limit of {MEMORY_NAME_LIMIT}."
+
+        timestamp = current_timestamp()
+        format_time = get_format_time()
+        # build create dict
+        memory_info = {
+            "id": get_uuid(),
+            "name": memory_name,
+            "memory_type": calculate_memory_type(memory_type),
+            "tenant_id": tenant_id,
+            "embd_id": embd_id,
+            "llm_id": llm_id,
+            "system_prompt": PromptAssembler.assemble_system_prompt({"memory_type": memory_type}),
+            "create_time": timestamp,
+            "create_date": format_time,
+            "update_time": timestamp,
+            "update_date": format_time,
+        }
+        obj = cls.model(**memory_info).save(force_insert=True)
+
+        if not obj:
+            return False, "Could not create new memory."
+
+        db_row = cls.model.select().where(cls.model.id == memory_info["id"]).first()
+
+        return obj, db_row
+
+    @classmethod
+    @DB.connection_context()
+    def update_memory(cls, tenant_id: str, memory_id: str, update_dict: dict):
+        if not update_dict:
+            return 0
+        if "temperature" in update_dict and isinstance(update_dict["temperature"], str):
+            update_dict["temperature"] = float(update_dict["temperature"])
+        if "memory_type" in update_dict and isinstance(update_dict["memory_type"], list):
+            update_dict["memory_type"] = calculate_memory_type(update_dict["memory_type"])
+        if "name" in update_dict:
+            update_dict["name"] = duplicate_name(
+                cls.query,
+                name=update_dict["name"],
+                tenant_id=tenant_id
+            )
+        update_dict.update({
+            "update_time": current_timestamp(),
+            "update_date": get_format_time()
+        })
+
+        return cls.model.update(update_dict).where(cls.model.id == memory_id).execute()
+
+    @classmethod
+    @DB.connection_context()
+    def delete_memory(cls, memory_id: str):
+        return cls.model.delete().where(cls.model.id == memory_id).execute()

api/db/services/pipeline_operation_log_service.py

@@ -121,7 +121,7 @@ class PipelineOperationLogService(CommonService):
         else:
             ok, kb_info = KnowledgebaseService.get_by_id(document.kb_id)
             if not ok:
-                raise RuntimeError(f"Cannot find knowledge base {document.kb_id} for referred_document {referred_document_id}")
+                raise RuntimeError(f"Cannot find dataset {document.kb_id} for referred_document {referred_document_id}")
 
             tenant_id = kb_info.tenant_id
             title = document.parser_id

api/db/services/task_service.py

@@ -76,7 +76,7 @@ class TaskService(CommonService):
         """Retrieve detailed task information by task ID.
 
         This method fetches comprehensive task details including associated document,
-        knowledge base, and tenant information. It also handles task retry logic and
+        dataset, and tenant information. It also handles task retry logic and
         progress updates.
 
         Args:
@@ -121,6 +121,13 @@ class TaskService(CommonService):
                 .where(cls.model.id == task_id)
         )
         docs = list(docs.dicts())
+        # Assuming docs = list(docs.dicts())
+        if docs:
+            kb_config = docs[0]['kb_parser_config']  # Dict from Knowledgebase.parser_config
+            mineru_method = kb_config.get('mineru_parse_method', 'auto')
+            mineru_formula = kb_config.get('mineru_formula_enable', True)
+            mineru_table = kb_config.get('mineru_table_enable', True)
+            print(mineru_method, mineru_formula, mineru_table)
         if not docs:
             return None
 

api/db/services/tenant_llm_service.py

@@ -14,15 +14,17 @@
 #  limitations under the License.
 #
 import os
+import json
 import logging
+from peewee import IntegrityError
 from langfuse import Langfuse
 from common import settings
-from common.constants import LLMType
+from common.constants import MINERU_DEFAULT_CONFIG, MINERU_ENV_KEYS, LLMType
 from api.db.db_models import DB, LLMFactories, TenantLLM
 from api.db.services.common_service import CommonService
 from api.db.services.langfuse_service import TenantLangfuseService
 from api.db.services.user_service import TenantService
-from rag.llm import ChatModel, CvModel, EmbeddingModel, RerankModel, Seq2txtModel, TTSModel
+from rag.llm import ChatModel, CvModel, EmbeddingModel, OcrModel, RerankModel, Seq2txtModel, TTSModel
 
 
 class LLMFactoriesService(CommonService):
@@ -95,7 +97,7 @@ class TenantLLMService(CommonService):
         if llm_type == LLMType.EMBEDDING.value:
             mdlnm = tenant.embd_id if not llm_name else llm_name
         elif llm_type == LLMType.SPEECH2TEXT.value:
-            mdlnm = tenant.asr_id
+            mdlnm = tenant.asr_id if not llm_name else llm_name
         elif llm_type == LLMType.IMAGE2TEXT.value:
             mdlnm = tenant.img2txt_id if not llm_name else llm_name
         elif llm_type == LLMType.CHAT.value:
@@ -104,6 +106,10 @@ class TenantLLMService(CommonService):
             mdlnm = tenant.rerank_id if not llm_name else llm_name
         elif llm_type == LLMType.TTS:
             mdlnm = tenant.tts_id if not llm_name else llm_name
+        elif llm_type == LLMType.OCR:
+            if not llm_name:
+                raise LookupError("OCR model name is required")
+            mdlnm = llm_name
         else:
             assert False, "LLM type error"
 
@@ -137,31 +143,31 @@ class TenantLLMService(CommonService):
             return EmbeddingModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"],
                                                                base_url=model_config["api_base"])
 
-        if llm_type == LLMType.RERANK:
+        elif llm_type == LLMType.RERANK:
             if model_config["llm_factory"] not in RerankModel:
                 return None
             return RerankModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"],
                                                             base_url=model_config["api_base"])
 
-        if llm_type == LLMType.IMAGE2TEXT.value:
+        elif llm_type == LLMType.IMAGE2TEXT.value:
             if model_config["llm_factory"] not in CvModel:
                 return None
             return CvModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"], lang,
                                                         base_url=model_config["api_base"], **kwargs)
 
-        if llm_type == LLMType.CHAT.value:
+        elif llm_type == LLMType.CHAT.value:
             if model_config["llm_factory"] not in ChatModel:
                 return None
             return ChatModel[model_config["llm_factory"]](model_config["api_key"], model_config["llm_name"],
                                                           base_url=model_config["api_base"], **kwargs)
 
-        if llm_type == LLMType.SPEECH2TEXT:
+        elif llm_type == LLMType.SPEECH2TEXT:
             if model_config["llm_factory"] not in Seq2txtModel:
                 return None
             return Seq2txtModel[model_config["llm_factory"]](key=model_config["api_key"],
                                                              model_name=model_config["llm_name"], lang=lang,
                                                              base_url=model_config["api_base"])
-        if llm_type == LLMType.TTS:
+        elif llm_type == LLMType.TTS:
             if model_config["llm_factory"] not in TTSModel:
                 return None
             return TTSModel[model_config["llm_factory"]](
@@ -169,6 +175,17 @@ class TenantLLMService(CommonService):
                 model_config["llm_name"],
                 base_url=model_config["api_base"],
             )
+
+        elif llm_type == LLMType.OCR:
+            if model_config["llm_factory"] not in OcrModel:
+                return None
+            return OcrModel[model_config["llm_factory"]](
+                key=model_config["api_key"],
+                model_name=model_config["llm_name"],
+                base_url=model_config.get("api_base", ""),
+                **kwargs,
+            )
+
         return None
 
     @classmethod
@@ -186,6 +203,7 @@ class TenantLLMService(CommonService):
             LLMType.CHAT.value: tenant.llm_id if not llm_name else llm_name,
             LLMType.RERANK.value: tenant.rerank_id if not llm_name else llm_name,
             LLMType.TTS.value: tenant.tts_id if not llm_name else llm_name,
+            LLMType.OCR.value: llm_name,
         }
 
         mdlnm = llm_map.get(llm_type)
@@ -218,6 +236,68 @@ class TenantLLMService(CommonService):
                                         ~(cls.model.llm_name == "text-embedding-3-large")).dicts()
         return list(objs)
 
+    @classmethod
+    def _collect_mineru_env_config(cls) -> dict | None:
+        cfg = MINERU_DEFAULT_CONFIG
+        found = False
+        for key in MINERU_ENV_KEYS:
+            val = os.environ.get(key)
+            if val:
+                found = True
+                cfg[key] = val
+        return cfg if found else None
+
+    @classmethod
+    @DB.connection_context()
+    def ensure_mineru_from_env(cls, tenant_id: str) -> str | None:
+        """
+        Ensure a MinerU OCR model exists for the tenant if env variables are present.
+        Return the existing or newly created llm_name, or None if env not set.
+        """
+        cfg = cls._collect_mineru_env_config()
+        if not cfg:
+            return None
+
+        saved_mineru_models = cls.query(tenant_id=tenant_id, llm_factory="MinerU", model_type=LLMType.OCR.value)
+
+        def _parse_api_key(raw: str) -> dict:
+            try:
+                return json.loads(raw or "{}")
+            except Exception:
+                return {}
+
+        for item in saved_mineru_models:
+            api_cfg = _parse_api_key(item.api_key)
+            normalized = {k: api_cfg.get(k, MINERU_DEFAULT_CONFIG.get(k)) for k in MINERU_ENV_KEYS}
+            if normalized == cfg:
+                return item.llm_name
+
+        used_names = {item.llm_name for item in saved_mineru_models}
+        idx = 1
+        base_name = "mineru-from-env"
+        while True:
+            candidate = f"{base_name}-{idx}"
+            if candidate in used_names:
+                idx += 1
+                continue
+
+            try:
+                cls.save(
+                    tenant_id=tenant_id,
+                    llm_factory="MinerU",
+                    llm_name=candidate,
+                    model_type=LLMType.OCR.value,
+                    api_key=json.dumps(cfg),
+                    api_base="",
+                    max_tokens=0,
+                )
+                return candidate
+            except IntegrityError:
+                logging.warning("MinerU env model %s already exists for tenant %s, retry with next name", candidate, tenant_id)
+                used_names.add(candidate)
+                idx += 1
+                continue
+
     @classmethod
     @DB.connection_context()
     def delete_by_tenant_id(cls, tenant_id):

api/ragflow_server.py

@@ -30,7 +30,7 @@ import threading
 import uuid
 import faulthandler
 
-from api.apps import app, smtp_mail_server
+from api.apps import app
 from api.db.runtime_config import RuntimeConfig
 from api.db.services.document_service import DocumentService
 from common.file_utils import get_project_base_directory
@@ -143,18 +143,6 @@ if __name__ == '__main__':
     else:
         threading.Timer(1.0, delayed_start_update_progress).start()
 
-    # init smtp server
-    if settings.SMTP_CONF:
-        app.config["MAIL_SERVER"] = settings.MAIL_SERVER
-        app.config["MAIL_PORT"] = settings.MAIL_PORT
-        app.config["MAIL_USE_SSL"] = settings.MAIL_USE_SSL
-        app.config["MAIL_USE_TLS"] = settings.MAIL_USE_TLS
-        app.config["MAIL_USERNAME"] = settings.MAIL_USERNAME
-        app.config["MAIL_PASSWORD"] = settings.MAIL_PASSWORD
-        app.config["MAIL_DEFAULT_SENDER"] = settings.MAIL_DEFAULT_SENDER
-        smtp_mail_server.init_app(app)
-
-
     # start http server
     try:
         logging.info("RAGFlow HTTP server start...")

api/utils/api_utils.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 #
 
+import asyncio
 import functools
 import inspect
 import json
@@ -25,7 +26,6 @@ from functools import wraps
 from typing import Any
 
 import requests
-import trio
 from quart import (
     Response,
     jsonify,
@@ -163,6 +163,7 @@ def validate_request(*args, **kwargs):
             if error_arguments:
                 error_string += "required argument values: {}".format(",".join(["{}={}".format(a[0], a[1]) for a in error_arguments]))
             return error_string
+        return None
 
     def wrapper(func):
         @wraps(func)
@@ -409,7 +410,7 @@ def get_parser_config(chunk_method, parser_config):
     if default_config is None:
         return deep_merge(base_defaults, parser_config)
 
-    # Ensure raptor and graphrag fields have default values if not provided
+    # Ensure raptor and graph_rag fields have default values if not provided
     merged_config = deep_merge(base_defaults, default_config)
     merged_config = deep_merge(merged_config, parser_config)
 
@@ -681,18 +682,32 @@ async def is_strong_enough(chat_model, embedding_model):
     async def _is_strong_enough():
         nonlocal chat_model, embedding_model
         if embedding_model:
-            with trio.fail_after(10):
-                _ = await trio.to_thread.run_sync(lambda: embedding_model.encode(["Are you strong enough!?"]))
+            await asyncio.wait_for(
+                asyncio.to_thread(embedding_model.encode, ["Are you strong enough!?"]),
+                timeout=10
+            )
+
         if chat_model:
-            with trio.fail_after(30):
-                res = await trio.to_thread.run_sync(lambda: chat_model.chat("Nothing special.", [{"role": "user", "content": "Are you strong enough!?"}], {}))
-            if res.find("**ERROR**") >= 0:
+            res = await asyncio.wait_for(
+                chat_model.async_chat("Nothing special.", [{"role": "user", "content": "Are you strong enough!?"}]),
+                timeout=30
+            )
+            if "**ERROR**" in res:
                 raise Exception(res)
 
     # Pressure test for GraphRAG task
-    async with trio.open_nursery() as nursery:
-        for _ in range(count):
-            nursery.start_soon(_is_strong_enough)
+    tasks = [
+        asyncio.create_task(_is_strong_enough())
+        for _ in range(count)
+    ]
+    try:
+        await asyncio.gather(*tasks, return_exceptions=False)
+    except Exception as e:
+        logging.error(f"Pressure test failed: {e}")
+        for t in tasks:
+            t.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+        raise
 
 
 def get_allowed_llm_factories() -> list:

api/utils/email_templates.py

@@ -20,18 +20,18 @@ Reusable HTML email templates and registry.
 
 # Invitation email template
 INVITE_EMAIL_TMPL = """
-<p>Hi {{email}},</p>
-<p>{{inviter}} has invited you to join their team (ID: {{tenant_id}}).</p>
-<p>Click the link below to complete your registration:<br>
-<a href="{{invite_url}}">{{invite_url}}</a></p>
-<p>If you did not request this, please ignore this email.</p>
+Hi {{email}},
+{{inviter}} has invited you to join their team (ID: {{tenant_id}}).
+Click the link below to complete your registration:
+{{invite_url}}
+If you did not request this, please ignore this email.
 """
 
 # Password reset code template
 RESET_CODE_EMAIL_TMPL = """
-<p>Hello,</p>
-<p>Your password reset code is: <b>{{ code }}</b></p>
-<p>This code will expire in {{ ttl_min }} minutes.</p>
+Hello,
+Your password reset code is: {{ code }}
+This code will expire in {{ ttl_min }} minutes.
 """
 
 # Template registry

api/utils/memory_utils.py

@@ -0,0 +1,54 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from typing import List
+from common.constants import MemoryType
+
+def format_ret_data_from_memory(memory):
+    return {
+        "id": memory.id,
+        "name": memory.name,
+        "avatar": memory.avatar,
+        "tenant_id": memory.tenant_id,
+        "owner_name": memory.owner_name if hasattr(memory, "owner_name") else None,
+        "memory_type": get_memory_type_human(memory.memory_type),
+        "storage_type": memory.storage_type,
+        "embd_id": memory.embd_id,
+        "llm_id": memory.llm_id,
+        "permissions": memory.permissions,
+        "description": memory.description,
+        "memory_size": memory.memory_size,
+        "forgetting_policy": memory.forgetting_policy,
+        "temperature": memory.temperature,
+        "system_prompt": memory.system_prompt,
+        "user_prompt": memory.user_prompt,
+        "create_time": memory.create_time,
+        "create_date": memory.create_date,
+        "update_time": memory.update_time,
+        "update_date": memory.update_date
+    }
+
+
+def get_memory_type_human(memory_type: int) -> List[str]:
+    return [mem_type.name.lower() for mem_type in MemoryType if memory_type & mem_type.value]
+
+
+def calculate_memory_type(memory_type_name_list: List[str]) -> int:
+    memory_type = 0
+    type_value_map = {mem_type.name.lower(): mem_type.value for mem_type in MemoryType}
+    for mem_type in memory_type_name_list:
+        if mem_type in type_value_map:
+            memory_type |= type_value_map[mem_type]
+    return memory_type

api/utils/web_utils.py

@@ -20,9 +20,10 @@ import json
 import re
 import socket
 from urllib.parse import urlparse
-
-from api.apps import smtp_mail_server
-from flask_mail import Message
+import aiosmtplib
+from email.mime.text import MIMEText
+from email.header import Header
+from common import settings
 from quart import render_template_string
 from api.utils.email_templates import EMAIL_TEMPLATES
 from selenium import webdriver
@@ -35,11 +36,11 @@ from selenium.webdriver.support.ui import WebDriverWait
 from webdriver_manager.chrome import ChromeDriverManager
 
 
-OTP_LENGTH = 8
-OTP_TTL_SECONDS = 5 * 60
-ATTEMPT_LIMIT = 5
-ATTEMPT_LOCK_SECONDS = 30 * 60
-RESEND_COOLDOWN_SECONDS = 60
+OTP_LENGTH = 4
+OTP_TTL_SECONDS = 5 * 60 # valid for 5 minutes
+ATTEMPT_LIMIT = 5 # maximum attempts
+ATTEMPT_LOCK_SECONDS = 30 * 60 # lock for 30 minutes
+RESEND_COOLDOWN_SECONDS = 60 # cooldown for 1 minute
 
 
 CONTENT_TYPE_MAP = {
@@ -183,27 +184,34 @@ def get_float(req: dict, key: str, default: float | int = 10.0) -> float:
         return parsed if parsed > 0 else default
     except (TypeError, ValueError):
         return default
+    
+
+async def send_email_html(to_email: str, subject: str, template_key: str, **context):
+
+    body = await render_template_string(EMAIL_TEMPLATES.get(template_key), **context)
+    msg = MIMEText(body, "plain", "utf-8")
+    msg["Subject"] = Header(subject, "utf-8")
+    msg["From"] = f"{settings.MAIL_DEFAULT_SENDER[0]} <{settings.MAIL_DEFAULT_SENDER[1]}>"
+    msg["To"] = to_email
+
+    smtp = aiosmtplib.SMTP(
+        hostname=settings.MAIL_SERVER,
+        port=settings.MAIL_PORT,
+        use_tls=True,
+        timeout=10,
+    )
+
+    await smtp.connect()
+    await smtp.login(settings.MAIL_USERNAME, settings.MAIL_PASSWORD)
+    await smtp.send_message(msg)
+    await smtp.quit()
 
 
-def send_email_html(subject: str, to_email: str, template_key: str, **context):
-    """Generic HTML email sender using shared templates.
-    template_key must exist in EMAIL_TEMPLATES.
-    """
-    from api.apps import app
-    tmpl = EMAIL_TEMPLATES.get(template_key)
-    if not tmpl:
-        raise ValueError(f"Unknown email template: {template_key}")
-    with app.app_context():
-        msg = Message(subject=subject, recipients=[to_email])
-        msg.html = render_template_string(tmpl, **context)
-        smtp_mail_server.send(msg)
-
-
-def send_invite_email(to_email, invite_url, tenant_id, inviter):
+async def send_invite_email(to_email, invite_url, tenant_id, inviter):
     # Reuse the generic HTML sender with 'invite' template
-    send_email_html(
-        subject="RAGFlow Invitation",
+    await send_email_html(
         to_email=to_email,
+        subject="RAGFlow Invitation",
         template_key="invite",
         email=to_email,
         invite_url=invite_url,
@@ -230,4 +238,4 @@ def hash_code(code: str, salt: bytes) -> str:
 
 def captcha_key(email: str) -> str:
     return f"captcha:{email}"
-    
+    

common/connection_utils.py

@@ -19,7 +19,6 @@ import queue
 import threading
 from typing import Any, Callable, Coroutine, Optional, Type, Union
 import asyncio
-import trio
 from functools import wraps
 from quart import make_response, jsonify
 from common.constants import RetCode
@@ -70,11 +69,10 @@ def timeout(seconds: float | int | str = None, attempts: int = 2, *, exception:
             for a in range(attempts):
                 try:
                     if os.environ.get("ENABLE_TIMEOUT_ASSERTION"):
-                        with trio.fail_after(seconds):
-                            return await func(*args, **kwargs)
+                        return await asyncio.wait_for(func(*args, **kwargs), timeout=seconds)
                     else:
                         return await func(*args, **kwargs)
-                except trio.TooSlowError:
+                except asyncio.TimeoutError:
                     if a < attempts - 1:
                         continue
                     if on_timeout is not None:

common/constants.py

@@ -54,6 +54,7 @@ class RetCode(IntEnum, CustomEnum):
     SERVER_ERROR = 500
     FORBIDDEN = 403
     NOT_FOUND = 404
+    CONFLICT = 409
 
 
 class StatusEnum(Enum):
@@ -73,6 +74,7 @@ class LLMType(StrEnum):
     IMAGE2TEXT = 'image2text'
     RERANK = 'rerank'
     TTS = 'tts'
+    OCR = 'ocr'
 
 
 class TaskStatus(StrEnum):
@@ -122,8 +124,13 @@ class FileSource(StrEnum):
     WEBDAV = "webdav"
     MOODLE = "moodle"
     DROPBOX = "dropbox"
+    BOX = "box"
+    R2 = "r2"
+    OCI_STORAGE = "oci_storage"
+    GOOGLE_CLOUD_STORAGE = "google_cloud_storage"
+    AIRTABLE = "airtable"
 
-
+    
 class PipelineTaskType(StrEnum):
     PARSE = "Parse"
     DOWNLOAD = "Download"
@@ -150,6 +157,23 @@ class Storage(Enum):
     OPENDAL = 6
     GCS = 7
 
+
+class MemoryType(Enum):
+    RAW = 0b0001          # 1 << 0 = 1 (0b00000001)
+    SEMANTIC = 0b0010     # 1 << 1 = 2 (0b00000010)
+    EPISODIC = 0b0100     # 1 << 2 = 4 (0b00000100)
+    PROCEDURAL = 0b1000   # 1 << 3 = 8 (0b00001000)
+
+
+class MemoryStorageType(StrEnum):
+    TABLE = "table"
+    GRAPH = "graph"
+
+
+class ForgettingPolicy(StrEnum):
+    FIFO = "FIFO"
+
+
 # environment
 # ENV_STRONG_TEST_COUNT = "STRONG_TEST_COUNT"
 # ENV_RAGFLOW_SECRET_KEY = "RAGFLOW_SECRET_KEY"
@@ -199,3 +223,13 @@ PAGERANK_FLD = "pagerank_fea"
 SVR_QUEUE_NAME = "rag_flow_svr_queue"
 SVR_CONSUMER_GROUP_NAME = "rag_flow_svr_task_broker"
 TAG_FLD = "tag_feas"
+
+
+MINERU_ENV_KEYS = ["MINERU_APISERVER", "MINERU_OUTPUT_DIR", "MINERU_BACKEND", "MINERU_SERVER_URL", "MINERU_DELETE_OUTPUT"]
+MINERU_DEFAULT_CONFIG = {
+    "MINERU_APISERVER": "",
+    "MINERU_OUTPUT_DIR": "",
+    "MINERU_BACKEND": "pipeline",
+    "MINERU_SERVER_URL": "",
+    "MINERU_DELETE_OUTPUT": 1,
+}

common/crypto_utils.py

@@ -0,0 +1,374 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import os
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.primitives import padding
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+from cryptography.hazmat.primitives import hashes
+
+
+class BaseCrypto:
+    """Base class for cryptographic algorithms"""
+    
+    # Magic header to identify encrypted data
+    ENCRYPTED_MAGIC = b'RAGF'
+    
+    def __init__(self, key, iv=None, block_size=16, key_length=32, iv_length=16):
+        """
+        Initialize cryptographic algorithm
+        
+        Args:
+            key: Encryption key
+            iv: Initialization vector, automatically generated if None
+            block_size: Block size
+            key_length: Key length
+            iv_length: Initialization vector length
+        """
+        self.block_size = block_size
+        self.key_length = key_length
+        self.iv_length = iv_length
+        
+        # Normalize key
+        self.key = self._normalize_key(key)
+        self.iv = iv
+    
+    def _normalize_key(self, key):
+        """Normalize key length"""
+        if isinstance(key, str):
+            key = key.encode('utf-8')
+        
+        # Use PBKDF2 for key derivation to ensure correct key length
+        kdf = PBKDF2HMAC(
+            algorithm=hashes.SHA256(),
+            length=self.key_length,
+            salt=b"ragflow_crypto_salt",  # Fixed salt to ensure consistent key derivation results
+            iterations=100000,
+            backend=default_backend()
+        )
+        
+        return kdf.derive(key)
+    
+    def encrypt(self, data):
+        """
+        Encrypt data (template method)
+        
+        Args:
+            data: Data to encrypt (bytes)
+            
+        Returns:
+            Encrypted data (bytes), format: magic_header + iv + encrypted_data
+        """
+        # Generate random IV
+        iv = os.urandom(self.iv_length) if not self.iv else self.iv
+        
+        # Use PKCS7 padding
+        padder = padding.PKCS7(self.block_size * 8).padder()
+        padded_data = padder.update(data) + padder.finalize()
+        
+        # Delegate to subclass for specific encryption
+        ciphertext = self._encrypt(padded_data, iv)
+        
+        # Return Magic Header + IV + encrypted data
+        return self.ENCRYPTED_MAGIC + iv + ciphertext
+    
+    def decrypt(self, encrypted_data):
+        """
+        Decrypt data (template method)
+        
+        Args:
+            encrypted_data: Encrypted data (bytes)
+            
+        Returns:
+            Decrypted data (bytes)
+        """
+        # Check if data is encrypted by magic header
+        if not encrypted_data.startswith(self.ENCRYPTED_MAGIC):
+            # Not encrypted, return as-is
+            return encrypted_data
+        
+        # Remove magic header
+        encrypted_data = encrypted_data[len(self.ENCRYPTED_MAGIC):]
+        
+        # Separate IV and encrypted data
+        iv = encrypted_data[:self.iv_length]
+        ciphertext = encrypted_data[self.iv_length:]
+        
+        # Delegate to subclass for specific decryption
+        padded_data = self._decrypt(ciphertext, iv)
+        
+        # Remove padding
+        unpadder = padding.PKCS7(self.block_size * 8).unpadder()
+        data = unpadder.update(padded_data) + unpadder.finalize()
+        
+        return data
+    
+    def _encrypt(self, padded_data, iv):
+        """
+        Encrypt padded data with specific algorithm
+        
+        Args:
+            padded_data: Padded data to encrypt
+            iv: Initialization vector
+            
+        Returns:
+            Encrypted data
+        """
+        raise NotImplementedError("_encrypt method must be implemented by subclass")
+    
+    def _decrypt(self, ciphertext, iv):
+        """
+        Decrypt ciphertext with specific algorithm
+        
+        Args:
+            ciphertext: Ciphertext to decrypt
+            iv: Initialization vector
+            
+        Returns:
+            Decrypted padded data
+        """
+        raise NotImplementedError("_decrypt method must be implemented by subclass")
+
+
+class AESCrypto(BaseCrypto):
+    """Base class for AES cryptographic algorithm"""
+    
+    def __init__(self, key, iv=None, key_length=32):
+        """
+        Initialize AES cryptographic algorithm
+        
+        Args:
+            key: Encryption key
+            iv: Initialization vector, automatically generated if None
+            key_length: Key length (16 for AES-128, 32 for AES-256)
+        """
+        super().__init__(key, iv, block_size=16, key_length=key_length, iv_length=16)
+
+    def _encrypt(self, padded_data, iv):
+        """AES encryption implementation"""
+        # Create encryptor
+        cipher = Cipher(
+            algorithms.AES(self.key),
+            modes.CBC(iv),
+            backend=default_backend()
+        )
+        encryptor = cipher.encryptor()
+        
+        # Encrypt data
+        return encryptor.update(padded_data) + encryptor.finalize()
+    
+    def _decrypt(self, ciphertext, iv):
+        """AES decryption implementation"""
+        # Create decryptor
+        cipher = Cipher(
+            algorithms.AES(self.key),
+            modes.CBC(iv),
+            backend=default_backend()
+        )
+        decryptor = cipher.decryptor()
+        
+        # Decrypt data
+        return decryptor.update(ciphertext) + decryptor.finalize()
+
+
+class AES128CBC(AESCrypto):
+    """AES-128-CBC cryptographic algorithm"""
+    
+    def __init__(self, key, iv=None):
+        """
+        Initialize AES-128-CBC cryptographic algorithm
+        
+        Args:
+            key: Encryption key
+            iv: Initialization vector, automatically generated if None
+        """
+        super().__init__(key, iv, key_length=16)
+
+
+class AES256CBC(AESCrypto):
+    """AES-256-CBC cryptographic algorithm"""
+    
+    def __init__(self, key, iv=None):
+        """
+        Initialize AES-256-CBC cryptographic algorithm
+        
+        Args:
+            key: Encryption key
+            iv: Initialization vector, automatically generated if None
+        """
+        super().__init__(key, iv, key_length=32)
+
+
+class SM4CBC(BaseCrypto):
+    """SM4-CBC cryptographic algorithm using cryptography library for better performance"""
+    
+    def __init__(self, key, iv=None):
+        """
+        Initialize SM4-CBC cryptographic algorithm
+        
+        Args:
+            key: Encryption key
+            iv: Initialization vector, automatically generated if None
+        """
+        super().__init__(key, iv, block_size=16, key_length=16, iv_length=16)
+
+    def _encrypt(self, padded_data, iv):
+        """SM4 encryption implementation using cryptography library"""
+        # Create encryptor
+        cipher = Cipher(
+            algorithms.SM4(self.key),
+            modes.CBC(iv),
+            backend=default_backend()
+        )
+        encryptor = cipher.encryptor()
+        
+        # Encrypt data
+        return encryptor.update(padded_data) + encryptor.finalize()
+    
+    def _decrypt(self, ciphertext, iv):
+        """SM4 decryption implementation using cryptography library"""
+        # Create decryptor
+        cipher = Cipher(
+            algorithms.SM4(self.key),
+            modes.CBC(iv),
+            backend=default_backend()
+        )
+        decryptor = cipher.decryptor()
+        
+        # Decrypt data
+        return decryptor.update(ciphertext) + decryptor.finalize()
+
+
+class CryptoUtil:
+    """Cryptographic utility class, using factory pattern to create cryptographic algorithm instances"""
+    
+    # Supported cryptographic algorithms mapping
+    SUPPORTED_ALGORITHMS = {
+        "aes-128-cbc": AES128CBC,
+        "aes-256-cbc": AES256CBC,
+        "sm4-cbc": SM4CBC
+    }
+    
+    def __init__(self, algorithm="aes-256-cbc", key=None, iv=None):
+        """
+        Initialize cryptographic utility
+        
+        Args:
+            algorithm: Cryptographic algorithm, default is aes-256-cbc
+            key: Encryption key, uses RAGFLOW_CRYPTO_KEY environment variable if None
+            iv: Initialization vector, automatically generated if None
+        """
+        if algorithm not in self.SUPPORTED_ALGORITHMS:
+            raise ValueError(f"Unsupported algorithm: {algorithm}")
+            
+        if not key:
+            raise ValueError("Encryption key not provided and RAGFLOW_CRYPTO_KEY environment variable not set")
+        
+        # Create cryptographic algorithm instance
+        self.algorithm_name = algorithm
+        self.crypto = self.SUPPORTED_ALGORITHMS[algorithm](key=key, iv=iv)
+    
+    def encrypt(self, data):
+        """
+        Encrypt data
+        
+        Args:
+            data: Data to encrypt (bytes)
+            
+        Returns:
+            Encrypted data (bytes)
+        """
+        # import time
+        # start_time = time.time()
+        encrypted = self.crypto.encrypt(data)
+        # end_time = time.time()
+        # logging.info(f"Encryption completed, data length: {len(data)} bytes, time: {(end_time - start_time)*1000:.2f} ms")
+        return encrypted
+    
+    def decrypt(self, encrypted_data):
+        """
+        Decrypt data
+        
+        Args:
+            encrypted_data: Encrypted data (bytes)
+            
+        Returns:
+            Decrypted data (bytes)
+        """
+        # import time
+        # start_time = time.time()
+        decrypted = self.crypto.decrypt(encrypted_data)
+        # end_time = time.time()
+        # logging.info(f"Decryption completed, data length: {len(encrypted_data)} bytes, time: {(end_time - start_time)*1000:.2f} ms")
+        return decrypted
+
+
+# Test code
+if __name__ == "__main__":
+    # Test AES encryption
+    crypto = CryptoUtil(algorithm="aes-256-cbc", key="test_key_123456")
+    test_data = b"Hello, RAGFlow! This is a test for encryption."
+    
+    encrypted = crypto.encrypt(test_data)
+    decrypted = crypto.decrypt(encrypted)
+    
+    print("AES Test:")
+    print(f"Original: {test_data}")
+    print(f"Encrypted: {encrypted}")
+    print(f"Decrypted: {decrypted}")
+    print(f"Success: {test_data == decrypted}")
+    print()
+    
+    # Test SM4 encryption
+    try:
+        crypto_sm4 = CryptoUtil(algorithm="sm4-cbc", key="test_key_123456")
+        encrypted_sm4 = crypto_sm4.encrypt(test_data)
+        decrypted_sm4 = crypto_sm4.decrypt(encrypted_sm4)
+        
+        print("SM4 Test:")
+        print(f"Original: {test_data}")
+        print(f"Encrypted: {encrypted_sm4}")
+        print(f"Decrypted: {decrypted_sm4}")
+        print(f"Success: {test_data == decrypted_sm4}")
+    except Exception as e:
+        print(f"SM4 Test Failed: {e}")
+        import traceback
+        traceback.print_exc()
+    
+    # Test with specific algorithm classes directly
+    print("\nDirect Algorithm Class Test:")
+    
+    # Test AES-128-CBC
+    aes128 = AES128CBC(key="test_key_123456")
+    encrypted_aes128 = aes128.encrypt(test_data)
+    decrypted_aes128 = aes128.decrypt(encrypted_aes128)
+    print(f"AES-128-CBC test: {'passed' if decrypted_aes128 == test_data else 'failed'}")
+    
+    # Test AES-256-CBC
+    aes256 = AES256CBC(key="test_key_123456")
+    encrypted_aes256 = aes256.encrypt(test_data)
+    decrypted_aes256 = aes256.decrypt(encrypted_aes256)
+    print(f"AES-256-CBC test: {'passed' if decrypted_aes256 == test_data else 'failed'}")
+    
+    # Test SM4-CBC
+    try:
+        sm4 = SM4CBC(key="test_key_123456")
+        encrypted_sm4 = sm4.encrypt(test_data)
+        decrypted_sm4 = sm4.decrypt(encrypted_sm4)
+        print(f"SM4-CBC test: {'passed' if decrypted_sm4 == test_data else 'failed'}")
+    except Exception as e:
+        print(f"SM4-CBC test failed: {e}")

common/data_source/__init__.py

@@ -1,6 +1,26 @@
 
 """
 Thanks to https://github.com/onyx-dot-app/onyx
+
+Content of this directory is under the "MIT Expat" license as defined below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
 """
 
 from .blob_connector import BlobStorageConnector
@@ -16,6 +36,7 @@ from .sharepoint_connector import SharePointConnector
 from .teams_connector import TeamsConnector
 from .webdav_connector import WebDAVConnector
 from .moodle_connector import MoodleConnector
+from .airtable_connector import AirtableConnector
 from .config import BlobType, DocumentSource
 from .models import Document, TextSection, ImageSection, BasicExpertInfo
 from .exceptions import (
@@ -50,5 +71,6 @@ __all__ = [
     "ConnectorValidationError",
     "CredentialExpiredError",
     "InsufficientPermissionsError",
-    "UnexpectedValidationError"
+    "UnexpectedValidationError",
+    "AirtableConnector",
 ]

common/data_source/airtable_connector.py

@@ -0,0 +1,149 @@
+from datetime import datetime, timezone
+import logging
+from typing import Any
+
+import requests
+
+from pyairtable import Api as AirtableApi
+
+from common.data_source.config import AIRTABLE_CONNECTOR_SIZE_THRESHOLD, INDEX_BATCH_SIZE, DocumentSource
+from common.data_source.exceptions import ConnectorMissingCredentialError
+from common.data_source.interfaces import LoadConnector
+from common.data_source.models import Document, GenerateDocumentsOutput
+from common.data_source.utils import extract_size_bytes, get_file_ext
+
+class AirtableClientNotSetUpError(PermissionError):
+    def __init__(self) -> None:
+        super().__init__(
+            "Airtable client is not set up. Did you forget to call load_credentials()?"
+        )
+
+
+class AirtableConnector(LoadConnector):
+    """
+    Lightweight Airtable connector.
+
+    This connector ingests Airtable attachments as raw blobs without
+    parsing file content or generating text/image sections.
+    """
+
+    def __init__(
+        self,
+        base_id: str,
+        table_name_or_id: str,
+        batch_size: int = INDEX_BATCH_SIZE,
+    ) -> None:
+        self.base_id = base_id
+        self.table_name_or_id = table_name_or_id
+        self.batch_size = batch_size
+        self._airtable_client: AirtableApi | None = None
+        self.size_threshold = AIRTABLE_CONNECTOR_SIZE_THRESHOLD
+
+    # -------------------------
+    # Credentials
+    # -------------------------
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
+        self._airtable_client = AirtableApi(credentials["airtable_access_token"])
+        return None
+
+    @property
+    def airtable_client(self) -> AirtableApi:
+        if not self._airtable_client:
+            raise AirtableClientNotSetUpError()
+        return self._airtable_client
+
+    # -------------------------
+    # Core logic
+    # -------------------------
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        """
+        Fetch all Airtable records and ingest attachments as raw blobs.
+
+        Each attachment is converted into a single Document(blob=...).
+        """
+        if not self._airtable_client:
+            raise ConnectorMissingCredentialError("Airtable credentials not loaded")
+
+        table = self.airtable_client.table(self.base_id, self.table_name_or_id)
+        records = table.all()
+
+        logging.info(
+            f"Starting Airtable blob ingestion for table {self.table_name_or_id}, "
+            f"{len(records)} records found."
+        )
+
+        batch: list[Document] = []
+
+        for record in records:
+            print(record)
+            record_id = record.get("id")
+            fields = record.get("fields", {})
+            created_time = record.get("createdTime")
+
+            for field_value in fields.values():
+                # We only care about attachment fields (lists of dicts with url/filename)
+                if not isinstance(field_value, list):
+                    continue
+
+                for attachment in field_value:
+                    url = attachment.get("url")
+                    filename = attachment.get("filename")
+                    attachment_id = attachment.get("id")
+
+                    if not url or not filename or not attachment_id:
+                        continue
+
+                    try:
+                        resp = requests.get(url, timeout=30)
+                        resp.raise_for_status()
+                        content = resp.content
+                    except Exception:
+                        logging.exception(
+                            f"Failed to download attachment {filename} "
+                            f"(record={record_id})"
+                        )
+                        continue
+                    size_bytes = extract_size_bytes(attachment)
+                    if (
+                        self.size_threshold is not None
+                        and isinstance(size_bytes, int)
+                        and size_bytes > self.size_threshold
+                    ):
+                        logging.warning(
+                            f"{filename} exceeds size threshold of {self.size_threshold}. Skipping."
+                        )
+                        continue
+                    batch.append(
+                        Document(
+                            id=f"airtable:{record_id}:{attachment_id}",
+                            blob=content,
+                            source=DocumentSource.AIRTABLE,
+                            semantic_identifier=filename,
+                            extension=get_file_ext(filename),
+                            size_bytes=size_bytes if size_bytes else 0,
+                            doc_updated_at=datetime.strptime(created_time, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
+                        )
+                    )
+
+                    if len(batch) >= self.batch_size:
+                        yield batch
+                        batch = []
+
+        if batch:
+            yield batch
+
+if __name__ == "__main__":
+    import os
+
+    logging.basicConfig(level=logging.DEBUG)
+    connector = AirtableConnector("xxx","xxx")
+    connector.load_credentials({"airtable_access_token": os.environ.get("AIRTABLE_ACCESS_TOKEN")})
+    connector.validate_connector_settings()
+    document_batches = connector.load_from_state()
+    try:
+        first_batch = next(document_batches)
+        print(f"Loaded {len(first_batch)} documents in first batch.")
+        for doc in first_batch:
+            print(f"- {doc.semantic_identifier} ({doc.size_bytes} bytes)")
+    except StopIteration:
+        print("No documents available in Dropbox.")

common/data_source/blob_connector.py

@@ -56,7 +56,7 @@ class BlobStorageConnector(LoadConnector, PollConnector):
 
         # Validate credentials
         if self.bucket_type == BlobType.R2:
-            if not all(
+            if not all( 
                 credentials.get(key)
                 for key in ["r2_access_key_id", "r2_secret_access_key", "account_id"]
             ):
@@ -64,15 +64,23 @@ class BlobStorageConnector(LoadConnector, PollConnector):
 
         elif self.bucket_type == BlobType.S3:
             authentication_method = credentials.get("authentication_method", "access_key")
+
             if authentication_method == "access_key":
                 if not all(
                     credentials.get(key)
                     for key in ["aws_access_key_id", "aws_secret_access_key"]
                 ):
                     raise ConnectorMissingCredentialError("Amazon S3")
+
             elif authentication_method == "iam_role":
                 if not credentials.get("aws_role_arn"):
                     raise ConnectorMissingCredentialError("Amazon S3 IAM role ARN is required")
+                
+            elif authentication_method == "assume_role":
+                pass
+
+            else:
+                raise ConnectorMissingCredentialError("Unsupported S3 authentication method")
 
         elif self.bucket_type == BlobType.GOOGLE_CLOUD_STORAGE:
             if not all(
@@ -120,55 +128,72 @@ class BlobStorageConnector(LoadConnector, PollConnector):
         paginator = self.s3_client.get_paginator("list_objects_v2")
         pages = paginator.paginate(Bucket=self.bucket_name, Prefix=self.prefix)
 
-        batch: list[Document] = []
+        # Collect all objects first to count filename occurrences
+        all_objects = []
         for page in pages:
             if "Contents" not in page:
                 continue
-
             for obj in page["Contents"]:
                 if obj["Key"].endswith("/"):
                     continue
-
                 last_modified = obj["LastModified"].replace(tzinfo=timezone.utc)
+                if start < last_modified <= end:
+                    all_objects.append(obj)
+        
+        # Count filename occurrences to determine which need full paths
+        filename_counts: dict[str, int] = {}
+        for obj in all_objects:
+            file_name = os.path.basename(obj["Key"])
+            filename_counts[file_name] = filename_counts.get(file_name, 0) + 1
 
-                if not (start < last_modified <= end):
+        batch: list[Document] = []
+        for obj in all_objects:
+            last_modified = obj["LastModified"].replace(tzinfo=timezone.utc)
+            file_name = os.path.basename(obj["Key"])
+            key = obj["Key"]
+
+            size_bytes = extract_size_bytes(obj)
+            if (
+                self.size_threshold is not None
+                and isinstance(size_bytes, int)
+                and size_bytes > self.size_threshold
+            ):
+                logging.warning(
+                    f"{file_name} exceeds size threshold of {self.size_threshold}. Skipping."
+                )
+                continue
+            
+            try:
+                blob = download_object(self.s3_client, self.bucket_name, key, self.size_threshold)
+                if blob is None:
                     continue
 
-                file_name = os.path.basename(obj["Key"])
-                key = obj["Key"]
+                # Use full path only if filename appears multiple times
+                if filename_counts.get(file_name, 0) > 1:
+                    relative_path = key
+                    if self.prefix and key.startswith(self.prefix):
+                        relative_path = key[len(self.prefix):]
+                    semantic_id = relative_path.replace('/', ' / ') if relative_path else file_name
+                else:
+                    semantic_id = file_name
 
-                size_bytes = extract_size_bytes(obj)
-                if (
-                    self.size_threshold is not None
-                    and isinstance(size_bytes, int)
-                    and size_bytes > self.size_threshold
-                ):
-                    logging.warning(
-                        f"{file_name} exceeds size threshold of {self.size_threshold}. Skipping."
+                batch.append(
+                    Document(
+                        id=f"{self.bucket_type}:{self.bucket_name}:{key}",
+                        blob=blob,
+                        source=DocumentSource(self.bucket_type.value),
+                        semantic_identifier=semantic_id,
+                        extension=get_file_ext(file_name),
+                        doc_updated_at=last_modified,
+                        size_bytes=size_bytes if size_bytes else 0
                     )
-                    continue
-                try:
-                    blob = download_object(self.s3_client, self.bucket_name, key, self.size_threshold)
-                    if blob is None:
-                        continue
+                )
+                if len(batch) == self.batch_size:
+                    yield batch
+                    batch = []
 
-                    batch.append(
-                        Document(
-                            id=f"{self.bucket_type}:{self.bucket_name}:{key}",
-                            blob=blob,
-                            source=DocumentSource(self.bucket_type.value),
-                            semantic_identifier=file_name,
-                            extension=get_file_ext(file_name),
-                            doc_updated_at=last_modified,
-                            size_bytes=size_bytes if size_bytes else 0
-                        )
-                    )
-                    if len(batch) == self.batch_size:
-                        yield batch
-                        batch = []
-
-                except Exception:
-                    logging.exception(f"Error decoding object {key}")
+            except Exception:
+                logging.exception(f"Error decoding object {key}")
         
         if batch:
             yield batch
@@ -276,4 +301,4 @@ if __name__ == "__main__":
     except ConnectorMissingCredentialError as e:
         print(f"Error: {e}")
     except Exception as e:
-        print(f"An unexpected error occurred: {e}")
+        print(f"An unexpected error occurred: {e}")

common/data_source/box_connector.py

@@ -0,0 +1,162 @@
+"""Box connector"""
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+from box_sdk_gen import BoxClient
+from common.data_source.config import DocumentSource, INDEX_BATCH_SIZE
+from common.data_source.exceptions import (
+    ConnectorMissingCredentialError,
+    ConnectorValidationError,
+)
+from common.data_source.interfaces import LoadConnector, PollConnector, SecondsSinceUnixEpoch
+from common.data_source.models import Document, GenerateDocumentsOutput
+from common.data_source.utils import get_file_ext
+
+class BoxConnector(LoadConnector, PollConnector):
+    def __init__(self, folder_id: str, batch_size: int = INDEX_BATCH_SIZE, use_marker: bool = True) -> None:
+        self.batch_size = batch_size
+        self.folder_id = "0" if not folder_id else folder_id
+        self.use_marker = use_marker
+        
+
+    def load_credentials(self, auth: Any):
+        self.box_client = BoxClient(auth=auth)
+        return None
+
+
+    def validate_connector_settings(self):
+        if self.box_client is None:
+            raise ConnectorMissingCredentialError("Box")
+
+        try:
+            self.box_client.users.get_user_me()
+        except Exception as e:
+            logging.exception("[Box]: Failed to validate Box credentials")
+            raise ConnectorValidationError(f"Unexpected error during Box settings validation: {e}")
+
+
+    def _yield_files_recursive(
+            self,
+            folder_id,
+            start: SecondsSinceUnixEpoch | None,
+            end: SecondsSinceUnixEpoch | None
+        ) -> GenerateDocumentsOutput:
+
+        if self.box_client is None:
+            raise ConnectorMissingCredentialError("Box")
+
+        result = self.box_client.folders.get_folder_items(
+            folder_id=folder_id,
+            limit=self.batch_size,
+            usemarker=self.use_marker
+        )
+
+        while True:
+            batch: list[Document] = []
+            for entry in result.entries:
+                if entry.type == 'file' :
+                    file = self.box_client.files.get_file_by_id(
+                        entry.id
+                    )
+                    raw_time = (
+                        getattr(file, "created_at", None)
+                        or getattr(file, "content_created_at", None)
+                    )
+
+                    if raw_time:
+                        modified_time = self._box_datetime_to_epoch_seconds(raw_time)
+                        if start is not None and modified_time <= start:
+                            continue
+                        if end is not None and modified_time > end:
+                            continue
+
+                    content_bytes = self.box_client.downloads.download_file(file.id)
+
+                    batch.append(
+                        Document(
+                            id=f"box:{file.id}",
+                            blob=content_bytes.read(),
+                            source=DocumentSource.BOX,
+                            semantic_identifier=file.name,
+                            extension=get_file_ext(file.name),
+                            doc_updated_at=modified_time,
+                            size_bytes=file.size,
+                            metadata=file.metadata
+                        )
+                    )
+                elif entry.type == 'folder':
+                    yield from self._yield_files_recursive(folder_id=entry.id, start=start, end=end)
+
+            if batch:
+                yield batch
+
+            if not result.next_marker:
+                break
+
+            result = self.box_client.folders.get_folder_items(
+                folder_id=folder_id,
+                limit=self.batch_size,
+                marker=result.next_marker,
+                usemarker=True
+            )
+
+
+    def _box_datetime_to_epoch_seconds(self, dt: datetime) -> SecondsSinceUnixEpoch:
+        """Convert a Box SDK datetime to Unix epoch seconds (UTC).
+        Only supports datetime; any non-datetime should be filtered out by caller.
+        """
+        if not isinstance(dt, datetime):
+            raise TypeError(f"box_datetime_to_epoch_seconds expects datetime, got {type(dt)}")
+
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        else:
+            dt = dt.astimezone(timezone.utc)
+
+        return SecondsSinceUnixEpoch(int(dt.timestamp()))
+
+
+    def poll_source(self, start, end):
+        return self._yield_files_recursive(folder_id=self.folder_id, start=start, end=end)
+
+
+    def load_from_state(self):
+        return self._yield_files_recursive(folder_id=self.folder_id, start=None, end=None)
+
+
+# from flask import Flask, request, redirect
+
+# from box_sdk_gen import BoxClient, BoxOAuth, OAuthConfig, GetAuthorizeUrlOptions
+
+# app = Flask(__name__)
+
+# AUTH = BoxOAuth(
+#     OAuthConfig(client_id="8suvn9ik7qezsq2dub0ye6ubox61081z", client_secret="QScvhLgBcZrb2ck1QP1ovkutpRhI2QcN")
+# )
+
+
+# @app.route("/")
+# def get_auth():
+#     auth_url = AUTH.get_authorize_url(
+#         options=GetAuthorizeUrlOptions(redirect_uri="http://localhost:4999/oauth2callback")
+#     )
+#     return redirect(auth_url, code=302)
+
+
+# @app.route("/oauth2callback")
+# def callback():
+#     AUTH.get_tokens_authorization_code_grant(request.args.get("code"))
+#     box = BoxConnector()
+#     box.load_credentials({"auth": AUTH})
+    
+#     lst = []
+#     for file in box.load_from_state():
+#        for f in file:
+#            lst.append(f.semantic_identifier)
+
+#     return lst
+
+if __name__ == "__main__":
+    pass
+    # app.run(port=4999)

common/data_source/config.py

@@ -52,7 +52,8 @@ class DocumentSource(str, Enum):
     MOODLE = "moodle"
     S3_COMPATIBLE = "s3_compatible"
     DROPBOX = "dropbox"
-
+    BOX = "box"
+    AIRTABLE = "airtable"
 
 class FileOrigin(str, Enum):
     """File origins"""
@@ -83,6 +84,7 @@ _PAGE_EXPANSION_FIELDS = [
     "space",
     "metadata.labels",
     "history.lastUpdated",
+    "ancestors",
 ]
 
 
@@ -227,6 +229,7 @@ _DEFAULT_PAGINATION_LIMIT = 1000
 _PROBLEMATIC_EXPANSIONS = "body.storage.value"
 _REPLACEMENT_EXPANSIONS = "body.view.value"
 
+BOX_WEB_OAUTH_REDIRECT_URI = os.environ.get("BOX_WEB_OAUTH_REDIRECT_URI", "http://localhost:9380/v1/connector/box/oauth/web/callback")
 
 class HtmlBasedConnectorTransformLinksStrategy(str, Enum):
     # remove links entirely
@@ -249,6 +252,10 @@ WEB_CONNECTOR_IGNORED_ELEMENTS = os.environ.get(
     "WEB_CONNECTOR_IGNORED_ELEMENTS", "nav,footer,meta,script,style,symbol,aside"
 ).split(",")
 
+AIRTABLE_CONNECTOR_SIZE_THRESHOLD = int(
+    os.environ.get("AIRTABLE_CONNECTOR_SIZE_THRESHOLD", 10 * 1024 * 1024)
+)
+
 _USER_NOT_FOUND = "Unknown Confluence User"
 
 _COMMENT_EXPANSION_FIELDS = ["body.storage.value"]

common/data_source/confluence_connector.py

@@ -126,7 +126,7 @@ class OnyxConfluence:
     def _renew_credentials(self) -> tuple[dict[str, Any], bool]:
         """credential_json - the current json credentials
         Returns a tuple
-        1. The up to date credentials
+        1. The up-to-date credentials
         2. True if the credentials were updated
 
         This method is intended to be used within a distributed lock.
@@ -179,14 +179,14 @@ class OnyxConfluence:
             credential_json["confluence_refresh_token"],
         )
 
-        # store the new credentials to redis and to the db thru the provider
-        # redis: we use a 5 min TTL because we are given a 10 minute grace period
+        # store the new credentials to redis and to the db through the provider
+        # redis: we use a 5 min TTL because we are given a 10 minutes grace period
         # when keys are rotated. it's easier to expire the cached credentials
         # reasonably frequently rather than trying to handle strong synchronization
         # between the db and redis everywhere the credentials might be updated
         new_credential_str = json.dumps(new_credentials)
         self.redis_client.set(
-            self.credential_key, new_credential_str, nx=True, ex=self.CREDENTIAL_TTL
+            self.credential_key, new_credential_str, exp=self.CREDENTIAL_TTL
         )
         self._credentials_provider.set_credentials(new_credentials)
 
@@ -690,7 +690,7 @@ class OnyxConfluence:
     ) -> Iterator[dict[str, Any]]:
         """
         This function will paginate through the top level query first, then
-        paginate through all of the expansions.
+        paginate through all the expansions.
         """
 
         def _traverse_and_update(data: dict | list) -> None:
@@ -717,7 +717,7 @@ class OnyxConfluence:
         """
         The search/user endpoint can be used to fetch users.
         It's a separate endpoint from the content/search endpoint used only for users.
-        Otherwise it's very similar to the content/search endpoint.
+        It's very similar to the content/search endpoint.
         """
 
         # this is needed since there is a live bug with Confluence Server/Data Center
@@ -863,7 +863,7 @@ def get_user_email_from_username__server(
             # For now, we'll just return None and log a warning. This means
             # we will keep retrying to get the email every group sync.
             email = None
-            # We may want to just return a string that indicates failure so we dont
+            # We may want to just return a string that indicates failure so we don't
             # keep retrying
             # email = f"FAILED TO GET CONFLUENCE EMAIL FOR {user_name}"
         _USER_EMAIL_CACHE[user_name] = email
@@ -912,7 +912,7 @@ def extract_text_from_confluence_html(
     confluence_object: dict[str, Any],
     fetched_titles: set[str],
 ) -> str:
-    """Parse a Confluence html page and replace the 'user Id' by the real
+    """Parse a Confluence html page and replace the 'user id' by the real
         User Display Name
 
     Args:
@@ -1110,7 +1110,10 @@ def _make_attachment_link(
 ) -> str | None:
     download_link = ""
 
-    if "api.atlassian.com" in confluence_client.url:
+    from urllib.parse import urlparse
+    netloc =urlparse(confluence_client.url).hostname
+    if netloc == "api.atlassian.com" or (netloc and netloc.endswith(".api.atlassian.com")):
+    # if "api.atlassian.com" in confluence_client.url:
         # https://developer.atlassian.com/cloud/confluence/rest/v1/api-group-content---attachments/#api-wiki-rest-api-content-id-child-attachment-attachmentid-download-get
         if not parent_content_id:
             logging.warning(
@@ -1308,6 +1311,9 @@ class ConfluenceConnector(
         self._low_timeout_confluence_client: OnyxConfluence | None = None
         self._fetched_titles: set[str] = set()
         self.allow_images = False
+        # Track document names to detect duplicates
+        self._document_name_counts: dict[str, int] = {}
+        self._document_name_paths: dict[str, list[str]] = {}
 
         # Remove trailing slash from wiki_base if present
         self.wiki_base = wiki_base.rstrip("/")
@@ -1510,6 +1516,40 @@ class ConfluenceConnector(
                 self.wiki_base, page["_links"]["webui"], self.is_cloud
             )
 
+            # Build hierarchical path for semantic identifier
+            space_name = page.get("space", {}).get("name", "")
+            
+            # Build path from ancestors
+            path_parts = []
+            if space_name:
+                path_parts.append(space_name)
+            
+            # Add ancestor pages to path if available
+            if "ancestors" in page and page["ancestors"]:
+                for ancestor in page["ancestors"]:
+                    ancestor_title = ancestor.get("title", "")
+                    if ancestor_title:
+                        path_parts.append(ancestor_title)
+            
+            # Add current page title
+            path_parts.append(page_title)
+            
+            # Track page names for duplicate detection
+            full_path = " / ".join(path_parts) if len(path_parts) > 1 else page_title
+            
+            # Count occurrences of this page title
+            if page_title not in self._document_name_counts:
+                self._document_name_counts[page_title] = 0
+                self._document_name_paths[page_title] = []
+            self._document_name_counts[page_title] += 1
+            self._document_name_paths[page_title].append(full_path)
+            
+            # Use simple name if no duplicates, otherwise use full path
+            if self._document_name_counts[page_title] == 1:
+                semantic_identifier = page_title
+            else:
+                semantic_identifier = full_path
+
             # Get the page content
             page_content = extract_text_from_confluence_html(
                 self.confluence_client, page, self._fetched_titles
@@ -1556,11 +1596,11 @@ class ConfluenceConnector(
             return Document(
                 id=page_url,
                 source=DocumentSource.CONFLUENCE,
-                semantic_identifier=page_title,
+                semantic_identifier=semantic_identifier,
                 extension=".html",  # Confluence pages are HTML
                 blob=page_content.encode("utf-8"),  # Encode page content as bytes
-                size_bytes=len(page_content.encode("utf-8")),  # Calculate size in bytes
                 doc_updated_at=datetime_from_string(page["version"]["when"]),
+                size_bytes=len(page_content.encode("utf-8")),  # Calculate size in bytes
                 primary_owners=primary_owners if primary_owners else None,
                 metadata=metadata if metadata else None,
             )
@@ -1598,7 +1638,6 @@ class ConfluenceConnector(
             expand=",".join(_ATTACHMENT_EXPANSION_FIELDS),
         ):
             media_type: str = attachment.get("metadata", {}).get("mediaType", "")
-
             # TODO(rkuo): this check is partially redundant with validate_attachment_filetype
             # and checks in convert_attachment_to_content/process_attachment
             # but doing the check here avoids an unnecessary download. Due for refactoring.
@@ -1666,6 +1705,34 @@ class ConfluenceConnector(
                     self.wiki_base, attachment["_links"]["webui"], self.is_cloud
                 )
 
+                # Build semantic identifier with space and page context
+                attachment_title = attachment.get("title", object_url)
+                space_name = page.get("space", {}).get("name", "")
+                page_title = page.get("title", "")
+                
+                # Create hierarchical name: Space / Page / Attachment
+                attachment_path_parts = []
+                if space_name:
+                    attachment_path_parts.append(space_name)
+                if page_title:
+                    attachment_path_parts.append(page_title)
+                attachment_path_parts.append(attachment_title)
+                
+                full_attachment_path = " / ".join(attachment_path_parts) if len(attachment_path_parts) > 1 else attachment_title
+                
+                # Track attachment names for duplicate detection
+                if attachment_title not in self._document_name_counts:
+                    self._document_name_counts[attachment_title] = 0
+                    self._document_name_paths[attachment_title] = []
+                self._document_name_counts[attachment_title] += 1
+                self._document_name_paths[attachment_title].append(full_attachment_path)
+                
+                # Use simple name if no duplicates, otherwise use full path
+                if self._document_name_counts[attachment_title] == 1:
+                    attachment_semantic_identifier = attachment_title
+                else:
+                    attachment_semantic_identifier = full_attachment_path
+
                 primary_owners: list[BasicExpertInfo] | None = None
                 if "version" in attachment and "by" in attachment["version"]:
                     author = attachment["version"]["by"]
@@ -1677,11 +1744,12 @@ class ConfluenceConnector(
 
                 extension = Path(attachment.get("title", "")).suffix or ".unknown"
 
+
                 attachment_doc = Document(
                     id=attachment_id,
                     # sections=sections,
                     source=DocumentSource.CONFLUENCE,
-                    semantic_identifier=attachment.get("title", object_url),
+                    semantic_identifier=attachment_semantic_identifier,
                     extension=extension,
                     blob=file_blob,
                     size_bytes=len(file_blob),
@@ -1738,7 +1806,7 @@ class ConfluenceConnector(
             start_ts, end, self.batch_size
         )
         logging.debug(f"page_query_url: {page_query_url}")
-
+        
         # store the next page start for confluence server, cursor for confluence cloud
         def store_next_page_url(next_page_url: str) -> None:
             checkpoint.next_page_url = next_page_url

common/data_source/discord_connector.py

@@ -33,7 +33,7 @@ def _convert_message_to_document(
     metadata: dict[str, str | list[str]] = {}
     semantic_substring = ""
 
-    # Only messages from TextChannels will make it here but we have to check for it anyways
+    # Only messages from TextChannels will make it here, but we have to check for it anyway
     if isinstance(message.channel, TextChannel) and (channel_name := message.channel.name):
         metadata["Channel"] = channel_name
         semantic_substring += f" in Channel: #{channel_name}"
@@ -176,7 +176,7 @@ def _manage_async_retrieval(
     # parse requested_start_date_string to datetime
     pull_date: datetime | None = datetime.strptime(requested_start_date_string, "%Y-%m-%d").replace(tzinfo=timezone.utc) if requested_start_date_string else None
 
-    # Set start_time to the later of start and pull_date, or whichever is provided
+    # Set start_time to the most recent of start and pull_date, or whichever is provided
     start_time = max(filter(None, [start, pull_date])) if start or pull_date else None
 
     end_time: datetime | None = end
@@ -233,8 +233,8 @@ class DiscordConnector(LoadConnector, PollConnector):
 
     def __init__(
         self,
-        server_ids: list[str] = [],
-        channel_names: list[str] = [],
+        server_ids: list[str] | None = None,
+        channel_names: list[str] | None = None,
         # YYYY-MM-DD
         start_date: str | None = None,
         batch_size: int = INDEX_BATCH_SIZE,